# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: PostgreSQL

# 33_build_memory_cleanup.pl — cleanup and lifecycle edges of the build-memory
# reservation the gate and its confirm/hand-off span do not otherwise cover:
# a FATAL from pg_terminate_backend mid-build, a rebuild landing on its new
# measured size rather than the sum of old and new, REINDEX CONCURRENTLY
# releasing the relid it replaces, and the reservation table's 64-slot
# ceiling with both flavors of its refusal.
#
# The reaper (ReapEntryReservations) is deliberately not exercised here: it
# only ever finds a RESERVED or REBUILDING record whose owning backend is
# dead while shared memory is still intact, and no live-cluster sequence
# reaches that state.  A clean ERROR, a FATAL, and pg_terminate_backend all
# unwind through SvsBuildAbortCleanup before the backend exits, releasing
# the reservation themselves; a SIGKILL instead makes the postmaster treat
# the death as possible corruption and force a full crash-restart, which
# reinitializes shared memory (including the reservation table) rather than
# leaving a dead owner's record behind.  Confirmed directly: SIGKILL
# against a backend parked at pre-confirm, holding a RESERVED reservation,
# produces "terminated by signal 9" followed by the postmaster's own
# crash-restart log line, and every other backend disconnects -- there is
# no surviving shared memory left to hold a stale entry in.

use strict;
use warnings FATAL => 'all';
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
use Time::HiRes qw(usleep);

use FindBin qw($Bin);
use lib "$Bin/../perl";
use VamanaTestUtils qw(:all);

if (($ENV{enable_injection_points} // 'no') ne 'yes')
{
	plan skip_all => 'server not built with --enable-injection-points';
}

my $node = PostgreSQL::Test::Cluster->new('build_memory_cleanup');
$node->init;
$node->append_conf('postgresql.conf', "shared_preload_libraries = 'vector,svs'");
$node->append_conf('postgresql.conf', "svs.launcher_database = 'postgres'");
$node->append_conf('postgresql.conf', "wal_level = logical");
$node->append_conf('postgresql.conf', "max_replication_slots = 80");
$node->append_conf('postgresql.conf', "max_wal_senders = 10");
$node->append_conf('postgresql.conf', "svs.max_build_memory = '4096MB'");
$node->append_conf('postgresql.conf', "svs.max_residency_memory = '100000MB'");
$node->append_conf('postgresql.conf', "svs.default_residency_memory = '16000MB'");
$node->append_conf('postgresql.conf', "svs.max_search_work_mem = '800MB'");
$node->start;

$node->safe_psql('postgres', "CREATE EXTENSION vector;");
$node->safe_psql('postgres', "CREATE EXTENSION svs;");
$node->safe_psql('postgres', "CREATE EXTENSION injection_points;");
$node->safe_psql('postgres',
	"INSERT INTO vamana_databases (datname, enabled) VALUES ('postgres', true);");
my $worker_pid = wait_for_worker($node);
ok($worker_pid =~ /^\d+$/, 'worker is running before the build-cleanup tests');

# worker_committed_totals and park_build are shared with
# 32_build_memory_gate.pl and live in VamanaTestUtils.pm.

# ---------------------------------------------------------------------------
# A terminated backend mid-build releases everything: pg_terminate_backend
# is a FATAL, and PG_ENSURE_ERROR_CLEANUP runs SvsBuildAbortCleanup on a
# FATAL exit the same as it does on an ordinary ERROR unwind.
# ---------------------------------------------------------------------------
{
	my $tbl = 'terminate_tbl';
	my $idx = 'terminate_idx';
	my $point = 'vamana-build-governed-pre-confirm';

	$node->safe_psql('postgres', qq(
		CREATE TABLE $tbl (id serial PRIMARY KEY, c1 vector($dim));
		INSERT INTO $tbl (c1)
			SELECT ARRAY[$array_sql]::vector FROM generate_series(1, 200) i;
	));

	my @before = worker_committed_totals($node, 'postgres');

	my $build = park_build($node, 'postgres', $tbl, $idx, $point);

	my $victim_pid = $node->safe_psql('postgres',
		"SELECT pid FROM pg_stat_activity WHERE wait_event = '$point';");
	chomp $victim_pid;
	like($victim_pid, qr/^\d+$/, 'terminate: found the backend parked at pre-confirm');

	# Terminate while still parked, holding the RESERVED reservation: waking
	# it first would let the build run to completion, including the
	# worker's own warm-up load, before the kill ever lands, which reaches
	# the out-of-scope leak this file's header describes instead of the
	# FATAL-mid-build path this case means to exercise.
	$node->safe_psql('postgres', "SELECT injection_points_detach('$point');");
	$node->safe_psql('postgres', "SELECT pg_terminate_backend($victim_pid);");

	# The terminated backend's own connection is now unusable; poll totals
	# from a fresh connection instead of querying $build.
	my @after;
	for my $i (1 .. 60)
	{
		@after = worker_committed_totals($node, 'postgres');
		last if "$after[0]|$after[1]" eq "$before[0]|$before[1]";
		usleep(500_000);
	}
	is_deeply(\@after, \@before,
		'terminate: pg_terminate_backend mid-build releases the build peak and residency reservation');

	eval { $build->quit };

	my $index_count = $node->safe_psql('postgres',
		"SELECT count(*) FROM pg_indexes WHERE indexname = '$idx';");
	chomp $index_count;
	is($index_count, '0', 'terminate: the terminated build left no index behind');

	my $live_pid = $node->safe_psql('postgres',
		"SELECT pid FROM pg_stat_activity WHERE backend_type = 'vamana worker';");
	chomp $live_pid;
	is($live_pid, $worker_pid, 'terminate: the worker survives the terminated build without restarting');
}

# ---------------------------------------------------------------------------
# REINDEX INDEX lands on the new measured size, not the sum of old and new:
# VamanaRebuildFromTable folds priorResidentBytes out before testing the
# fresh measurement, rather than adding the new estimate on top of what was
# already committed.  This path runs VamanaCacheIndex synchronously in the
# rebuilding backend itself, unlike a fresh CREATE INDEX's asynchronous
# hand-off to the worker, so the residency record is already correct by the
# time REINDEX INDEX returns -- no polling needed.
# ---------------------------------------------------------------------------
{
	my $tbl = 'reindex_size_tbl';
	my $idx = 'reindex_size_idx';

	$node->safe_psql('postgres', qq(
		CREATE TABLE $tbl (id serial PRIMARY KEY, c1 vector($dim));
		INSERT INTO $tbl (c1)
			SELECT ARRAY[$array_sql]::vector FROM generate_series(1, 5000) i;
		CREATE INDEX $idx ON $tbl USING vamana (c1 vector_l2_ops) WITH (graph_degree = 16);
	));

	my $relid = $node->safe_psql('postgres', "SELECT '$idx'::regclass::oid;");
	chomp $relid;
	my $old_bytes = $node->safe_psql('postgres',
		"SELECT resident_bytes FROM svs_index_residency WHERE index_relid = $relid;");
	chomp $old_bytes;
	cmp_ok($old_bytes, '>', 0, 'reindex_size: the original build reports a plausible resident size');

	my (undef, $prior_total) = worker_committed_totals($node, 'postgres');

	# A materially denser graph over the same 5000 rows, so the rebuild's
	# measured size is not a coin flip against the original's.
	$node->safe_psql('postgres', "ALTER INDEX $idx SET (graph_degree = 256);");
	$node->safe_psql('postgres', "REINDEX INDEX $idx;");

	my $new_bytes = $node->safe_psql('postgres',
		"SELECT resident_bytes FROM svs_index_residency WHERE index_relid = $relid;");
	chomp $new_bytes;
	cmp_ok($new_bytes, '!=', $old_bytes,
		'reindex_size: the denser rebuild measures a materially different size');

	my (undef, $after_total) = worker_committed_totals($node, 'postgres');
	is($after_total, $prior_total - $old_bytes + $new_bytes,
		'reindex_size: the committed total is the prior total minus the old size plus the new one, not their sum');

	my $row_count = $node->safe_psql('postgres',
		"SELECT count(*) FROM svs_index_residency WHERE index_relid = $relid;");
	chomp $row_count;
	is($row_count, '1', 'reindex_size: exactly one residency row for this relid');
}

# ---------------------------------------------------------------------------
# REINDEX INDEX CONCURRENTLY builds a fresh index under a new relid and
# drops the old one; the committed total must end up counting only the new
# index's bytes, with no leftover residency row for the relid it replaced.
# Both halves are asynchronous -- the new index through the same worker
# hand-off a fresh CREATE INDEX takes, the old one through the worker's own
# eviction on DROP -- so both are polled rather than read immediately.
# ---------------------------------------------------------------------------
{
	my $tbl = 'reindex_conc_tbl';
	my $idx = 'reindex_conc_idx';

	$node->safe_psql('postgres', qq(
		CREATE TABLE $tbl (id serial PRIMARY KEY, c1 vector($dim));
		INSERT INTO $tbl (c1)
			SELECT ARRAY[$array_sql]::vector FROM generate_series(1, 2000) i;
		CREATE INDEX $idx ON $tbl USING vamana (c1 vector_l2_ops);
	));

	my $old_relid = $node->safe_psql('postgres', "SELECT '$idx'::regclass::oid;");
	chomp $old_relid;

	my $old_bytes;
	for my $i (1 .. 60)
	{
		$old_bytes = $node->safe_psql('postgres',
			"SELECT resident_bytes FROM svs_index_residency WHERE index_relid = $old_relid;");
		chomp $old_bytes;
		last if $old_bytes =~ /^\d+$/ && $old_bytes > 0;
		usleep(500_000);
	}
	cmp_ok($old_bytes, '>', 0, 'reindex_concurrently: the original build becomes resident');

	my (undef, $prior_total) = worker_committed_totals($node, 'postgres');

	$node->safe_psql('postgres', "REINDEX INDEX CONCURRENTLY $idx;");

	my $new_relid = $node->safe_psql('postgres', "SELECT '$idx'::regclass::oid;");
	chomp $new_relid;
	isnt($new_relid, $old_relid,
		'reindex_concurrently: the concurrent reindex assigns a new relid, not the old one');

	my $new_bytes;
	for my $i (1 .. 60)
	{
		$new_bytes = $node->safe_psql('postgres',
			"SELECT resident_bytes FROM svs_index_residency WHERE index_relid = $new_relid;");
		chomp $new_bytes;
		last if $new_bytes =~ /^\d+$/ && $new_bytes > 0;
		usleep(500_000);
	}
	cmp_ok($new_bytes, '>', 0, 'reindex_concurrently: the replacement index becomes resident');

	my (undef, $after_total) = worker_committed_totals($node, 'postgres');
	is($after_total, $prior_total - $old_bytes + $new_bytes,
		'reindex_concurrently: the committed total ends up counting only the replacement index, '
	  . 'not the one it replaced on top of it')
	  or diag("prior_total=$prior_total old_bytes=$old_bytes new_bytes=$new_bytes after_total=$after_total");

	my $old_relid_row_count = $node->safe_psql('postgres',
		"SELECT count(*) FROM svs_index_residency WHERE index_relid = $old_relid;");
	chomp $old_relid_row_count;
	is($old_relid_row_count, '0',
		'reindex_concurrently: no leftover residency row for the relid it replaced');
}

# ---------------------------------------------------------------------------
# Slot exhaustion at VAMANA_MAX_INDEXES (64), shared by build and load
# reservations for one database.  A dedicated database keeps this count
# exact, uncontaminated by any index another case in this file created.
# ---------------------------------------------------------------------------
{
	my $db = 'slot_exhaustion';

	$node->safe_psql('postgres', "CREATE DATABASE $db;");
	$node->safe_psql($db, "CREATE EXTENSION vector;");
	$node->safe_psql($db, "CREATE EXTENSION svs;");
	$node->safe_psql('postgres',
		"INSERT INTO vamana_databases (datname, enabled) VALUES ('$db', true);");
	wait_for_worker_db($node, $db);

	my $n_slots = 64;
	for my $i (1 .. $n_slots)
	{
		$node->safe_psql($db, qq(
			CREATE TABLE slot_tbl_$i (id serial PRIMARY KEY, c1 vector($dim));
			INSERT INTO slot_tbl_$i (c1)
				SELECT ARRAY[$array_sql]::vector FROM generate_series(1, 5) s;
			CREATE INDEX slot_idx_$i ON slot_tbl_$i USING vamana (c1 vector_l2_ops);
		));
	}
	my $built_count = $node->safe_psql($db,
		"SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'slot_idx_%';");
	chomp $built_count;
	is($built_count, "$n_slots", "slot_exhaustion: all $n_slots tiny indexes are resident");

	$node->safe_psql($db, qq(
		CREATE TABLE slot_tbl_over (id serial PRIMARY KEY, c1 vector($dim));
		INSERT INTO slot_tbl_over (c1)
			SELECT ARRAY[$array_sql]::vector FROM generate_series(1, 5) s;
	));

	my ($ret_over, $stdout_over, $stderr_over) = $node->psql($db,
		"CREATE INDEX slot_idx_over ON slot_tbl_over USING vamana (c1 vector_l2_ops);");
	isnt($ret_over, 0,
		"slot_exhaustion: the ${\ ($n_slots + 1)}th index is refused once all $n_slots slots are resident");
	like($stderr_over, qr/too many concurrently tracked SVS indexes/,
		'slot_exhaustion: the refusal names the tracking limit')
	  or diag("stderr: $stderr_over");
	like($stderr_over, qr/Reduce the number of indexes on this database, or accept the limit\./,
		'slot_exhaustion: the errhint is the at-capacity one, since every slot is resident, not building')
	  or diag("stderr: $stderr_over");

	my $over_count = $node->safe_psql($db,
		"SELECT count(*) FROM pg_indexes WHERE indexname = 'slot_idx_over';");
	chomp $over_count;
	is($over_count, '0', 'slot_exhaustion: the refused build left no index behind');

	# memLock is released on refusal, not held: drop one resident index and
	# confirm a new build succeeds once the worker's eviction frees its slot.
	$node->safe_psql($db, "DROP INDEX slot_idx_1;");

	my $retry_succeeded = 0;
	my ($ret_retry, $stdout_retry, $stderr_retry);
	for my $i (1 .. 60)
	{
		($ret_retry, $stdout_retry, $stderr_retry) = $node->psql($db,
			"CREATE INDEX slot_idx_over ON slot_tbl_over USING vamana (c1 vector_l2_ops);");
		if ($ret_retry == 0)
		{
			$retry_succeeded = 1;
			last;
		}
		usleep(500_000);
	}
	ok($retry_succeeded,
		'slot_exhaustion: memLock is free again -- a new build succeeds once the dropped slot is reclaimed')
	  or diag("stderr: $stderr_retry");

	# Blocked-by-builds variant (AllocateReservation's other errhint, for
	# slots held by in-progress builds rather than resident indexes):
	# attempted and abandoned as impractical in this environment, per this
	# case's own instructions. residency_bytes_committed's own timing near
	# this boundary was not reliably reproducible run to run, and a parked
	# reproduction attempt separately observed a build admitted into a
	# RESERVED reservation while pg_stat_vamana_worker reported an
	# index_count of 62, with no way in this file to determine whether true
	# reservation-table occupancy was actually below 64 at that instant. Not
	# filed as a confirmed defect; flagged here for whoever next touches
	# slot accounting near this boundary.
}

$node->stop;

done_testing();