/*-------------------------------------------------------------------------
 *
 * pg_fts_analyze.c
 *		Stage-1 built-in tokenizer for pg_fts.
 *
 * Produces an ftsdoc from raw text.  This is the simple, self-contained default
 * analyzer -- to_ftsdoc(text): fold ASCII letters to lowercase, split on any
 * non-alphanumeric byte, and collect the distinct terms with their term
 * frequencies.
 *
 * The configuration-driven analyzer that reuses PostgreSQL's text-search parser
 * and dictionary pipeline (parsetext(), the snowball/ispell dictionaries) lives
 * in pg_fts_tsanalyze.c as to_ftsdoc(regconfig, text).  Tokenization is isolated
 * behind fts_analyze_text() so either analyzer can be used without touching the
 * type, the operator, or the on-disk format.
 *
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
 *
 * IDENTIFICATION
 *	  pg_fts_analyze.c
 *
 *-------------------------------------------------------------------------
 */
#include "postgres.h"

#include "pg_fts.h"
#include "catalog/pg_collation.h"
#include "common/unicode_case.h"
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "utils/formatting.h"
#include "utils/memutils.h"

PG_MODULE_MAGIC;

/* One collected token before we fold duplicates into the ftsdoc. */
typedef struct RawTerm
{
	char	   *term;
	int			len;
	uint32		pos;			/* 1-based token position in the source */
} RawTerm;

/*
 * Case-fold a single ASCII byte.  This is the fast path used by fold_token()
 * for ASCII-only tokens; non-ASCII tokens go through Unicode lowercasing.
 */
static inline char
fold_ascii(unsigned char c)
{
	if (c >= 'A' && c <= 'Z')
		return (char) (c - 'A' + 'a');
	return (char) c;
}

static inline bool
is_token_byte(unsigned char c)
{
	/* ASCII alphanumerics start a/continue a token; so do all non-ASCII bytes
	 * (so UTF-8 words are kept whole rather than split at every byte). */
	if (c >= 0x80)
		return true;
	return (c >= 'a' && c <= 'z') ||
		(c >= 'A' && c <= 'Z') ||
		(c >= '0' && c <= '9');
}

/*
 * An ASCII-only token takes the length-preserving byte fast path.  In a UTF-8
 * database, a token containing any byte >= 0x80 is lowercased per Unicode code
 * point via unicode_lowercase_simple(), so 'E'+U+0301 style caseful pairs like
 * 'É'/'é' fold together and a query for one matches the other.
 *
 * *outlen - receives the folded byte length.
 */
char *
fold_token(const char *src, int len, int *outlen)
{
	char	   *dst = (char *) palloc(Max(len, 1));
	char	   *out;
	const unsigned char *srcp;
	const unsigned char *srcend;
	int			i = 0;

	/*
	 * Fast path: fold ASCII bytes in a single pass, stopping at the first
	 * non-ASCII byte.  ASCII case-folding equals Unicode lowercasing for those
	 * bytes, so a prefix folded here stays correct even if we fall through to
	 * the Unicode path.
	 */
	while (i < len && (unsigned char) src[i] < 0x80)
	{
		dst[i] = fold_ascii((unsigned char) src[i]);
		i++;
	}
	if (i == len)
	{
		*outlen = len;			/* pure ASCII: one pass, exact-sized buffer */
		return dst;
	}

	if (GetDatabaseEncoding() != PG_UTF8)
	{
		/*
		 * Non-UTF-8 server encoding.  ASCII bytes were already folded above; the
		 * remaining high bytes (>= 0x80) are single-byte or multibyte characters
		 * of the server encoding, which we cannot interpret as Unicode code
		 * points.  Delegate to str_tolower() -- the same locale/collation-aware
		 * primitive PostgreSQL's own text search uses -- so a LATIN1/WIN1252
		 * database with a real locale folds accented letters exactly as
		 * to_tsvector() does.
		 *
		 * Without this, a non-UTF-8 server left high bytes UNCHANGED, so on e.g.
		 * LATIN1 + de_DE a document 'Apfel' (A-umlaut 0xC4) did not match a query
		 * 'apfel' (a-umlaut 0xE4): case-insensitive search silently failed for
		 * every non-ASCII letter, diverging from PostgreSQL's behaviour.
		 *
		 * str_tolower() may change the byte length (some encodings/locales are not
		 * length-preserving), so take its result rather than writing in place.
		 * DEFAULT_COLLATION_OID matches what tsearch's lowerstr() effectively
		 * uses (the database default), keeping us consistent with to_tsvector.
		 */
		char	   *low = str_tolower(src, (size_t) len, DEFAULT_COLLATION_OID);
		int			lowlen = (int) strlen(low);

		pfree(dst);
		*outlen = lowlen;
		return low;
	}

	/*
	 * UTF-8: lowercase the remaining code points.  The folded length can now
	 * differ from len, so grow the buffer to the worst case (each code point's
	 * simple lowercase is a single code point, at most 4 UTF-8 bytes); the
	 * already-folded ASCII prefix is preserved by repalloc.
	 */
	dst = (char *) repalloc(dst, (Size) len * 4 + 1);
	out = dst + i;
	srcp = (const unsigned char *) src + i;
	srcend = (const unsigned char *) src + len;
	while (srcp < srcend)
	{
		int			clen = pg_utf_mblen(srcp);
		pg_wchar	lc;

		/* Defensive: never read past the token.  The tokenizer keeps whole UTF-8
		 * characters together (is_token_byte treats every >= 0x80 byte as part of
		 * a token), so a truncated trailing character should not reach here -- but
		 * utf8_to_unicode() would read clen bytes regardless, so bound it and stop
		 * rather than over-read if any future caller hands us a split token. */
		if (srcp + clen > srcend)
			break;
		lc = unicode_lowercase_simple(utf8_to_unicode(srcp));

		/* unicode_to_utf8() returns the START pointer, so advance out by the
		 * code point's UTF-8 length ourselves. */
		unicode_to_utf8(lc, (unsigned char *) out);
		out += unicode_utf8len(lc);
		srcp += clen;
	}
	*outlen = (int) (out - dst);
	return dst;
}

static int
cmp_rawterm(const void *a, const void *b)
{
	const RawTerm *ra = (const RawTerm *) a;
	const RawTerm *rb = (const RawTerm *) b;
	int			min = Min(ra->len, rb->len);
	int			c = memcmp(ra->term, rb->term, min);

	if (c != 0)
		return c;
	if (ra->len != rb->len)
		return ra->len - rb->len;
	/* same term: order by position so positions come out ascending */
	if (ra->pos < rb->pos)
		return -1;
	if (ra->pos > rb->pos)
		return 1;
	return 0;
}

/*
 * fts_analyze_text -- tokenize raw text into an ftsdoc.
 *
 * Returns a palloc'd, fully formed FtsDoc varlena.  An empty input yields a
 * valid zero-term document (which matches nothing).
 */
FtsDoc
fts_analyze_text(const char *str, int len)
{
	RawTerm    *raw;
	int			nraw = 0;
	int			maxraw;
	int			i;
	uint32		doclen = 0;

	/* Upper bound on tokens: every other byte could start a token.  Size with a
	 * huge-safe alloc: a very large single document (e.g. an inline patch series)
	 * can push maxraw*sizeof(RawTerm) past MaxAllocSize, and a plain palloc would
	 * throw "invalid memory alloc request size" mid-analyze. */
	maxraw = (len / 2) + 1;
	raw = (RawTerm *) (((Size) maxraw * sizeof(RawTerm)) > MaxAllocSize
					   ? MemoryContextAllocHuge(CurrentMemoryContext,
												(Size) maxraw * sizeof(RawTerm))
					   : palloc((Size) maxraw * sizeof(RawTerm)));	/* alloc-ok: huge branch of the > MaxAllocSize ternary above */

	i = 0;
	while (i < len)
	{
		int			start;

		/* skip separators */
		while (i < len && !is_token_byte((unsigned char) str[i]))
			i++;
		if (i >= len)
			break;

		start = i;
		while (i < len && is_token_byte((unsigned char) str[i]))
			i++;

		Assert(nraw < maxraw);
		raw[nraw].term = fold_token(str + start, i - start, &raw[nraw].len);
		raw[nraw].pos = doclen + 1;	/* 1-based token position */
		nraw++;
		doclen++;
	}

	/* Sort by (term, pos) so duplicates are adjacent and positions ascend. */
	if (nraw > 1)
		qsort(raw, nraw, sizeof(RawTerm), cmp_rawterm);

	/* Second pass: fold duplicates, recording tf and positions. */
	{
		int			ndistinct = 0;
		Size		lexbytes = 0;
		int			npos = nraw;	/* one position per token */
		FtsDoc		doc;
		Size		posbase;
		Size		total;
		FtsTermEntry *entries;
		char	   *lexemes;
		uint32	   *positions;
		uint32		off;
		uint32		pidx;
		int		   *runlen;
		int		   *runstart;

		/* identify distinct-term runs (term-only equality) */
		runlen = (int *) palloc(Max(nraw, 1) * sizeof(int));
		runstart = (int *) palloc(Max(nraw, 1) * sizeof(int));
		for (i = 0; i < nraw;)
		{
			int			run = 1;

			while (i + run < nraw)
			{
				int			min = Min(raw[i].len, raw[i + run].len);

				if (raw[i].len != raw[i + run].len ||
					memcmp(raw[i].term, raw[i + run].term, min) != 0)
					break;
				run++;
			}
			runstart[ndistinct] = i;
			runlen[ndistinct] = run;
			lexbytes += raw[i].len;
			ndistinct++;
			i += run;
		}

		posbase = MAXALIGN(FTS_DOC_HDRSIZE +
						   (Size) ndistinct * sizeof(FtsTermEntry) + lexbytes);
		total = posbase + (Size) npos * sizeof(uint32);
		if (total > MaxAllocSize)
			ereport(ERROR,
					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
					 errmsg("ftsdoc document is too large"),
					 errdetail("An ftsdoc value is limited to %zu bytes; this document needs %zu.",
							   (Size) MaxAllocSize, total)));
		doc = (FtsDoc) palloc0(total);
		SET_VARSIZE(doc, total);
		doc->version = FTS_DOC_VERSION;
		doc->flags = FTS_DOCF_POSITIONS;
		doc->nterms = ndistinct;
		doc->doclen = doclen;
		doc->lexbytes = lexbytes;

		entries = FTS_DOC_ENTRIES(doc);
		lexemes = FTS_DOC_LEXEMES(doc);
		positions = FTS_DOC_POSITIONS(doc);
		off = 0;
		pidx = 0;
		for (i = 0; i < ndistinct; i++)
		{
			int			s = runstart[i];
			int			k;

			entries[i].off = off;
			entries[i].len = raw[s].len;
			entries[i].tf = runlen[i];
			entries[i].posoff = pidx;
			memcpy(lexemes + off, raw[s].term, raw[s].len);
			off += raw[s].len;
			for (k = 0; k < runlen[i]; k++)
				positions[pidx++] = raw[s + k].pos;
		}

		return doc;
	}
}

PG_FUNCTION_INFO_V1(to_ftsdoc);

Datum
to_ftsdoc(PG_FUNCTION_ARGS)
{
	text	   *in = PG_GETARG_TEXT_PP(0);
	FtsDoc		doc;

	doc = fts_analyze_text(VARDATA_ANY(in), VARSIZE_ANY_EXHDR(in));

	PG_FREE_IF_COPY(in, 0);
	PG_RETURN_FTSDOC(doc);
}