#!/usr/bin/env bash
# Throwaway PostgreSQL cluster that loads sift from the repo build output, without a
# system-wide install (extension_control_path, dynamic_library_path). PostgreSQL 17 has no
# extension_control_path: install then copies sift into the server's own directories, as
# SIFT_SUDO (e.g. sudo) when those are not writable. Two checkouts that install for PostgreSQL 17 on one machine overwrite each other's library there (a test then fails with an
# option or a behavior of the other checkout): use PostgreSQL 18 when several work on a machine, or install, start and run in one go.
#
#   scripts/pgtest.sh install    build + install into $SIFT_TMP/root
#   scripts/pgtest.sh start      initdb (first time) + start, without autovacuum (see start)
#   scripts/pgtest.sh psql [...] psql into the test database
#   scripts/pgtest.sh env        print exports for libpq clients (eval "$(...)")
#   scripts/pgtest.sh conf       print the postgresql.conf lines that make sift loadable
#   scripts/pgtest.sh regress [t...]  make installcheck against the cluster; with test names, pg_regress runs just those,
#                                in one database per call (contrib_regression_<first>) and an output directory $SIFT_TMP/regress/<t> per test (parallel
#                                runs); a test that leaves objects behind fails (scripts/leftovers.sql, exceptions in scripts/leftovers.allow)
#                                Run the tests that compare heap-free reads (col_heapfree_*, col_track_*, col_tomb_*, col_stmt_cid*) one at a time on a quiet cluster: a session
#                                of another test holds back the xmin horizon, and what a read may take from the log without the heap (free_reads_heap) changes by design
#   scripts/pgtest.sh fresh [t...]    regress with a NEW backend for every statement outside a transaction (scripts/fresh-psql.py, test/fresh/README.md)
#   scripts/pgtest.sh leftovers [t...]  each test alone in a database of its own; lists what it left behind (scripts/leftovers.sql)
#   scripts/pgtest.sh tap [t]    TAP shell tests test/t/*.sh (or test/t/<t>.sh), own clusters
#   scripts/pgtest.sh ready      wait until the cluster accepts connections (recovery after a crashed backend)
#   scripts/pgtest.sh clean      drop the databases a job left in the cluster (all but postgres and the test database), once it is ready
#   scripts/pgtest.sh stop       stop the cluster
#   scripts/pgtest.sh destroy    stop and remove $SIFT_TMP
#   scripts/pgtest.sh sanitizer-report  print ASan/UBSan findings (exit 1 if any)
#   scripts/pgtest.sh initdb-template DIR   the data directory SIFT_INITDB_TEMPLATE points at (1 MB WAL segments)
#   scripts/pgtest.sh new-data DIR [wal]    a fresh data directory (the template copied, else initdb); wal: the default 16 MB WAL
#                                           segments, for the tests whose point is WAL (always a real initdb)
#
# SIFT_TMP (default /tmp/sift-arch) and SIFT_PORT (56100-56299, default
# 56100) let several jobs run clusters side by side; SIFT_ROOT (default
# $SIFT_TMP/root) lets clusters in other SIFT_TMP directories share one install; TAP clusters use
# SIFT_PORT + 1 and up. SIFT_TESTING=1 (default) builds the sift._test_*
# functions; SIFT_TESTING= builds the release library.
#
# VULKAN=1 install builds the opt-in Vulkan backend (docs/GPU.md).
# SANITIZE=1 install builds with ASan+UBSan; SIFT_HARDEN=pkg|init|1 builds with the hardening flags of scripts/harden.mk
# (docs/SAFETY.md; SIFT_POINTER_OVERFLOW=1 adds the pointer-wrap trap); install runs make clean whenever SANITIZE, SIFT_TESTING or SIFT_HARDEN changes. start then preloads the sanitizer runtimes
# into the server, with LeakSanitizer (scripts/lsan.supp); reports go to
# $SIFT_TMP/sanitizer/ (sanitizer-report). SIFT_INITDB_TEMPLATE, a data directory made by `initdb-template`, is copied
# instead of running initdb for every cluster (scripts/ci.sh makes one per run, test/t/lib.bash uses it too).
# Every cluster is under the leak gate of scripts/leakcheck.sh: start records the memory of the
# long-lived processes, stop checks the cluster, stops it cleanly (fast, so that LeakSanitizer
# reports) and scans the logs. Exit 1 on a finding.
set -euo pipefail

REPO=$(cd "$(dirname "$0")/.." && pwd)
# macOS: the postmaster dies with "became multithreaded" unless LC_ALL names a valid locale (the default C of a non-login shell does not count)
[[ $(uname) != Darwin ]] || export LC_ALL=en_US.UTF-8
TMP=${SIFT_TMP:-/tmp/sift-arch}
PORT=${SIFT_PORT:-56100}
DB=${SIFT_DB:-sift}
ROOT=${SIFT_ROOT:-$TMP/root}
export SIFT_ROOT=$ROOT
DATA=$TMP/data
SOCK=$TMP/sock
SANDIR=$TMP/sanitizer
SHAREDIR=$ROOT$(pg_config --sharedir)
PKGLIBDIR=$ROOT$(pg_config --pkglibdir)
. "$(dirname "$0")/leakcheck.sh"
PG_MAJOR=$(pg_config --version | sed -n 's/^PostgreSQL \([0-9][0-9]*\).*/\1/p')
# the cluster is this one and no other: a default of the caller (a service, a database) must not send a client to the system PostgreSQL
unset PGDATABASE PGSERVICE PGSERVICEFILE
export PGHOST=$SOCK PGPORT=$PORT
export SIFT_TESTING=${SIFT_TESTING-1}
FLAVOR="${SANITIZE:+asan }${SIFT_TESTING:+testing }${SIFT_HARDEN:+harden$SIFT_HARDEN${SIFT_UBSAN_SET:+(${SIFT_UBSAN_SET// /,})}${SIFT_POINTER_OVERFLOW:++ptr} }$([[ -n ${SIFT_LTO:-} ]] && echo lto || echo nolto)${VULKAN:+ vulkan} pg$PG_MAJOR"

(( PORT >= 56100 && PORT <= 56299 )) || { echo "SIFT_PORT must be in 56100-56299" >&2; exit 1; }

DLSUFFIX=.so; [[ $(uname) != Darwin ]] || DLSUFFIX=.dylib   # the suffix of a PostgreSQL module

install() {
	echo "pgtest: install: $([[ -n $SIFT_TESTING ]] && echo 'test library' || echo 'release library') (flavor ${FLAVOR% })" >&2   # SIFT_TESTING set and empty is the release library: an inherited empty value builds it
	[[ $(cat "$REPO/.build-flavor" 2>/dev/null) == "$FLAVOR" ]] || make -C "$REPO" USE_PGXS=1 clean >/dev/null
	# on the make command line: sudo resets the environment, which would install another flavor
	local flavor=(USE_PGXS=1 SIFT_TESTING="$SIFT_TESTING" SANITIZE="${SANITIZE:-}" SIFT_LTO="${SIFT_LTO:-}" SIFT_HARDEN="${SIFT_HARDEN:-}" ${SIFT_UBSAN_SET:+"SIFT_UBSAN_SET=$SIFT_UBSAN_SET"} ${SIFT_POINTER_OVERFLOW:+SIFT_POINTER_OVERFLOW=1} ${VULKAN:+VULKAN=1})
	nice make -C "$REPO" -j4 "${flavor[@]}"
	"$REPO/scripts/check-undefined.sh" "$REPO/pg_sift$DLSUFFIX"
	echo "$FLAVOR" >"$REPO/.build-flavor"
	if (( PG_MAJOR >= 18 )); then
		make -C "$REPO" "${flavor[@]}" DESTDIR="$ROOT" install >/dev/null
	else
		${SIFT_SUDO:-} make -C "$REPO" "${flavor[@]}" install >/dev/null
		mkdir -p "$ROOT"
	fi
	echo "$FLAVOR" >"$ROOT/.flavor"
}

# The lines that make the staged sift loadable (the server's own install on PostgreSQL 17)
conf() {
	leak_conf
	# the background pass of a sift_table is on by default; a test counts the segments, blocks and dead rows its own statements leave, so the
	# clusters run without it as without autovacuum, and the tests of the pass switch it on
	echo "sift.table_auto_defrag = off"
	# likewise the worker that vacuums the tables of sift_col indexes and rewrites the indexes (on by default)
	echo "sift.col_auto_merge = off"
	# and a test that switches it on wants its rewrite when the rule says so, not when the cluster has written enough (the test of the budget sets its own)
	echo "sift.col_auto_merge_wal_pct = 0"
	# the tests of the options that 4.3 deprecated build with them until 4.4 removes them; the tests of the default say SET sift.test_col_legacy_options = off
	echo "sift.test_col_legacy_options = on"
	# a server with LLVM compiles every plan above jit_above_cost, and the tests that force a plan with page costs of 10000 reach it: the normal plan of 1,200
	# partitions took 5.6 s to compile and 11 ms to run (PostgreSQL 17 of the CI runners has the library in its package, 18 in another one)
	echo "jit = off"
	# 19 can aggregate below a join (eager aggregation) and prices that plan under some of those of the pushdown tests, which then keep the normal plan
	(( PG_MAJOR < 19 )) || echo "enable_eager_aggregate = off"
	(( PG_MAJOR >= 18 )) || return 0
	echo "extension_control_path = '$SHAREDIR:\$system'"
	echo "dynamic_library_path = '$PKGLIBDIR:\$libdir'"
}

# The server binary is not instrumented, so the ASan runtime must be preloaded.
sanitizer_env() { [[ $(cat "$ROOT/.flavor" 2>/dev/null) != asan* ]] || leak_sanitizer_env "$SANDIR"; }

# The cluster every test starts from: superuser postgres, UTF8, no locale. The clusters run with fsync off, so initdb need not sync
# either. WAL segments of 1 MB (the default 16 MB is a floor of 64 MB of pg_wal per cluster) except for KIND wal.
initdb_template() {
	local seg=(--wal-segsize=1)
	[[ ${2:-} != wal ]] || seg=()
	initdb --no-sync -D "$1" -A trust -U postgres -E UTF8 --no-locale --no-instructions "${seg[@]}" >/dev/null
}

# new_data DIR [wal]: a fresh data directory, copied from SIFT_INITDB_TEMPLATE when there is one
new_data() {
	if [[ -n ${SIFT_INITDB_TEMPLATE:-} && ${2:-} != wal ]]; then cp -a "$SIFT_INITDB_TEMPLATE" "$1"; else initdb_template "$1" "${2:-}"; fi
}

# No autovacuum in the cluster: a worker holds a snapshot while it analyzes a table, a catalog too, and VACUUM removes only the rows older
# than every snapshot of the database, so the rows a test deleted stay and its plan or its dead-document count differs now and then (the
# reloption autovacuum_enabled cannot switch off the workers' catalogs). The one job that needs it (scripts/soak.sh) turns it on for its run.
# max_wal_size 512MB: with 48MB a test that writes 30 MB of WAL in 14 s (col_sorted_run) checkpointed all the time, and on a runner disk that wrote
# 0.1 MB/s a backend waited 100 to 250 s for a dirty buffer (33 to 255 "checkpoints are occurring too frequently" per lane in a PostgreSQL 19 quick shard).
# The WAL a lane keeps is at most this, 1 MB segments, no recycling.
# reset_auto_conf DIR: the settings an ALTER SYSTEM of an earlier session left (max_parallel_workers_per_gather = 0, a small shared_buffers, work_mem) change the plans of
# the tests: 18 of them failed on a clean tree for it. The cluster starts from the conf above alone.
reset_auto_conf() {
	if grep -qv '^#' "$1/postgresql.auto.conf" 2>/dev/null; then
		echo "pgtest: $1/postgresql.auto.conf held settings from ALTER SYSTEM; emptied" >&2
		: >"$1/postgresql.auto.conf"
	fi
}

start() {
	mkdir -p "$SOCK"
	if [[ ! -f $DATA/PG_VERSION ]]; then
		new_data "$DATA"
		cat >>"$DATA/postgresql.conf" <<-EOF
			port = $PORT
			listen_addresses = ''
			unix_socket_directories = '$SOCK'
			$(conf)
			fsync = off
			max_wal_size = 512MB
			min_wal_size = 80MB
			wal_recycle = off
			shared_buffers = 64MB
			max_worker_processes = 16
			max_locks_per_transaction = 256
			max_prepared_transactions = 8
			autovacuum = off
		EOF
	fi
	reset_auto_conf "$DATA"
	( sanitizer_env; pg_ctl -D "$DATA" -l "$TMP/postgres.log" -w start >/dev/null )
	createdb -U postgres "$DB" 2>/dev/null || true
	leak_baseline "$DATA"
}

stop() {
	[[ -f $DATA/postmaster.pid ]] || return 0
	local rc=0
	PGUSER=postgres leak_cluster "$DATA" || rc=1
	pg_ctl -D "$DATA" -m fast -w stop >/dev/null
	leak_after_stop "$DATA" || rc=1
	leak_scan "$TMP" || rc=1
	return $rc
}

# ready: waits until the cluster accepts connections (SIFT_READY_SECS, 300). A backend that dies (a sanitizer that halts it, a PANIC test) makes the postmaster
# recover every other session; under ASan that takes a minute, and a job that starts in it fails at once with "the database system is not yet accepting connections"
ready() {
	local i
	for ((i = 0; i < ${SIFT_READY_SECS:-300} * 2; i++)); do
		pg_isready -q -h "$SOCK" -p "$PORT" && return 0
		sleep 0.5
	done
	echo "pgtest: the cluster at $SOCK does not accept connections after ${SIFT_READY_SECS:-300} s" >&2
	return 1
}

# clean: drops every database but the template ones, postgres and the test database: what a job created
clean() {
	local d
	ready || return 1
	for d in $(psql -U postgres -d postgres -X -qAt -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'template0', 'template1', '$DB')"); do
		dropdb -U postgres --force "$d"
	done
}

# regress [TEST...]: without names, make installcheck (the Makefile's database and test list). With names, one pg_regress run per test in one
# database for all, each followed by a look at what it left in the database (scripts/leftovers.sql): a test that leaves something fails the run,
# unless scripts/leftovers.allow gives the object and the reason, and the next test starts in a fresh database so that the one that left it is
# the only failure and not also those that find it there
regress() {
	if (( ! $# )); then make -C "$REPO" USE_PGXS=1 installcheck PGUSER=postgres; return; fi
	local db=contrib_regression_$1 t out rc=0 reuse=() before= now left
	for t in "$@"; do
		out=$TMP/regress/$t
		if ! pg_regress "$out" "$db" "${reuse[@]}" "$t"; then
			rc=1; cat "$out/regression.diffs" 2>/dev/null; reuse=() before=
			continue
		fi
		now=$(db_left "$db")
		left=$({ grep -vxFf <(echo "$before") <<<"$now" || true; } | { grep -vEf <(sed -n "s/^$t: \(.*[^ ]\) *# .*/\1/p" "$REPO/scripts/leftovers.allow") || true; })
		if [[ -n $left ]]; then
			rc=1; reuse=() before=
			sed "s/^/LEFTOVER $t: /" <<<"$left"
			echo "not ok - $t leaves objects behind: end it with the DROPs of what it creates (the extension last), or give the object and the reason in scripts/leftovers.allow"
		else
			reuse=(--use-existing) before=$now
		fi
	done
	return $rc
}

# db_left DB: what scripts/leftovers.sql finds in DB (the objects of the database, not those of the cluster)
db_left() { psql -U postgres -d "$1" -X -qAt -v cluster=off -f "$REPO/scripts/leftovers.sql"; }

# pg_regress OUTDIR DB TEST...: pg_regress in a database it creates, the output in a fresh OUTDIR
pg_regress() {
	local out=$1 db=$2
	shift 2
	rm -rf "$out" && mkdir -p "$out"
	# SIFT_FRESH: a psql that connects anew before every statement outside a transaction (scripts/fresh-psql.py) stands first in PATH, in this subshell only
	# (FRESH_PSQL is the real psql, found before PATH changes: the shim refuses to run as its own psql or inside itself)
	(
		if [[ -n ${SIFT_FRESH:-} ]]; then
			mkdir -p "$TMP/fresh-bin" && ln -sf "$REPO/scripts/fresh-psql.py" "$TMP/fresh-bin/psql"
			FRESH_PSQL=$(command -v psql)
			export FRESH_PSQL FRESH_STATS=$out/fresh-stats PATH=$TMP/fresh-bin:$PATH
		fi
		PGUSER=postgres "$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress" --inputdir="$REPO/test" --outputdir="$out" --bindir= --dbname="$db" "$@"
	)
}

# leftovers [TEST...]: each test alone in a database of its own (all of test/sql without names); prints "TEST: object" for what it left behind
# (scripts/leftovers.sql: relations, functions, types, schemas, extensions, roles, ALTER SYSTEM...) and exits 1 when any test left something
leftovers() {
	local t db bad=0 before now left
	(( $# )) || set -- $(cd "$REPO/test/sql" && ls *.sql | sed 's/\.sql$//')
	cluster_left() { psql -U postgres -d postgres -X -qAt -v cluster=on -f "$REPO/scripts/leftovers.sql" | { grep -E '^(role|ALTER SYSTEM) ' || true; }; }
	before=$(cluster_left)
	for t in "$@"; do
		db=contrib_regression_$t
		pg_regress "$TMP/regress/$t" "$db" "$t" >/dev/null 2>&1 || echo "$t: FAILED, what it left may come from that (diffs in $TMP/regress/$t)"
		now=$(cluster_left)
		left=$(db_left "$db"; grep -vxFf <(echo "$before") <<<"$now" || true)
		[[ -z ${left//$'\n'/} ]] || { bad=1; echo "$left" >"$TMP/regress/$t/leftovers"; sed "s/^/$t: /" <<<"$left" | head -12; (( $(wc -l <<<"$left") <= 12 )) || echo "$t: ... all $(wc -l <<<"$left") in $TMP/regress/$t/leftovers"; }
		before=$now
		dropdb -U postgres --force "$db"
	done
	return $bad
}

tap() {
	local tests=(test/t/*.sh)
	[[ -n $1 ]] && tests=(test/t/$1.sh)
	cd "$REPO"
	# the sanitizer runtime goes into the servers (SIFT_PRELOAD) and not into prove, bash or psql
	( sanitizer_env; SIFT_PRELOAD=${LD_PRELOAD:-}; unset LD_PRELOAD; export SIFT_PRELOAD
	  SIFT_TMP=$TMP SIFT_ROOT=$ROOT SIFT_PORT=$PORT prove ${PROVE_FLAGS:---failures --comments} --exec bash "${tests[@]}" )
}

case ${1:-} in
	install) install ;;
	start) start ;;
	stop) stop ;;
	conf) conf ;;
	psql) shift; exec psql -U postgres -d "$DB" -X "$@" ;;
	env)
		echo "export PGHOST=$SOCK PGPORT=$PORT PGUSER=postgres SIFT_ROOT=$ROOT"
		sanitizer_env
		[[ -z ${ASAN_OPTIONS:-} ]] || echo "export SIFT_SANITIZER='LD_PRELOAD=$LD_PRELOAD ASAN_OPTIONS=$ASAN_OPTIONS LSAN_OPTIONS=$LSAN_OPTIONS UBSAN_OPTIONS=$UBSAN_OPTIONS'"
		;;
	regress) shift; regress "$@" ;;
	fresh)
		shift
		# at most SIFT_FRESH_RUNS (default 1, at most 4) fresh runs on a host (every statement is a connection), and never inside one (the shim stands first in PATH of the pg_regress subshell only)
		[[ -z ${FRESH_PSQL:-}${FRESH_ACTIVE:-} ]] || { echo "pgtest: fresh runs do not nest" >&2; exit 1; }
		slot=0 got=
		for ((slot = 0; slot < ${SIFT_FRESH_RUNS:-1} && slot < 4; slot++)); do
			exec 9>"${XDG_RUNTIME_DIR:-/tmp}/sift-fresh.$slot.lock"
			if flock -n 9; then got=1; break; fi
		done
		[[ -n $got ]] || { echo "pgtest: ${SIFT_FRESH_RUNS:-1} fresh run(s) are at work (SIFT_FRESH_RUNS, at most 4)" >&2; exit 1; }
		SIFT_FRESH=1 regress "$@" ;;
	leftovers) shift; leftovers "$@" ;;
	reset-auto-conf) reset_auto_conf "$2" ;;
	initdb-template) initdb_template "$2" ;;
	new-data) new_data "$2" "${3:-}" ;;
	tap) tap "${2:-}" ;;
	ready) ready ;;
	clean) clean ;;
	destroy) stop || true; rm -rf "$TMP" ;;
	sanitizer-report) leak_scan "$SANDIR" ;;
	*) sed -n "2,32p" "$0"; exit 1 ;;
esac