# kill -9 of the backend running an index build, a maintenance step (GIN
# pending-list cleanup / sift promotion and merge) or VACUUM, then crash
# recovery, verification and result equivalence with the SQL reference.
#
# Not shrunk below this size: the kill has to land inside the operation, so each operation runs long enough
# to be killed part way, and each kill point costs about 4.5 s (the kill, crash recovery, verification and the
# agreement with the SQL reference). With fewer rows (8,000 where this file takes 18 s) the operation ends
# before the kill and the file passes without having crashed anything.
use strict;
use warnings FATAL => 'all';
use FindBin;
use lib $FindBin::RealBin;
use SiftTest;
use Test::More;

my $rows = knob('SIFT_ROWS', 60000, 20000);
my $torn = knob('SIFT_TORN_ROWS', 300000, 40000);	# an insert still running 150 ms in, at any speed
my $late = knob('SIFT_LATE_ROWS', 15000, 6000);	# rows the maintenance step has to promote or merge
my $batch = knob('SIFT_BATCH_ROWS', 40000, 15000);	# rows of a merge batch: dozens of segments at the 64 kB write_buffer_limit
my $node = sift_node('crash');
$node->start;
install_sift($node);

for my $am (ams())
{
	my $n = $am->{name};
	subtest "$n: crash during build, maintenance and vacuum" => sub {
		my ($t, $idx) = ("crash_$n", "crash_${n}_idx");
		my $hits = 0;
		load_corpus($node, $t, $rows);

		for my $delay (0, 100)
		{
			$hits += crash_during($node, $am->{index}->($t, $idx), $delay);
			$node->safe_psql('postgres', "DROP INDEX IF EXISTS $idx");
		}
		$node->safe_psql('postgres', $am->{index}->($t, $idx));
		$node->safe_psql('postgres', $am->{verify}->($idx));
		agree($node, $am, $t, 'rebuilt after crashed builds');

		# a bulk insert through aminsert, killed mid-flight: its rows must vanish from the index with the abort
		$hits += crash_during($node, "INSERT INTO $t SELECT g, 'w6 rare torn ' || g FROM generate_series(300001, 300000 + $torn) g", 150);
		$node->safe_psql('postgres', $am->{verify}->($idx));
		agree($node, $am, $t, 'crash in insert');

		for my $delay (0, 50)
		{
			$node->safe_psql('postgres', "INSERT INTO $t SELECT g, 'w1 rare late ' || g"
				. " FROM generate_series(100001 + $delay * 1000, 100000 + $late + $delay * 1000) g");
			$hits += crash_during($node, $am->{maint}->($idx), $delay);
			$node->safe_psql('postgres', $am->{verify}->($idx));
			agree($node, $am, $t, "crash in maintenance after $delay ms");
		}

		if ($n eq 'sift')
		{
			for my $b (1 .. 3)
			{
				# a 64 kB write_buffer_limit seals dozens of segments per batch: keep room below the 128 segment cap
				$node->safe_psql('postgres', "SELECT sift.merge('$idx')");
				$node->safe_psql('postgres', "INSERT INTO $t SELECT g, 'w6 merge batch$b ' || g"
					. " FROM generate_series(1000000 + $b * 100000, 999999 + $batch + $b * 100000) g;"
					. " SELECT sift.promote('$idx')");
			}
			$hits += crash_during($node, "SELECT sift.merge('$idx')", 5);
			$node->safe_psql('postgres', $am->{verify}->($idx));
			agree($node, $am, $t, 'crash in merge');
		}

		$node->safe_psql('postgres', "DELETE FROM $t WHERE id % 3 = 0");
		$hits += crash_during($node, "VACUUM $t", 30);
		$node->safe_psql('postgres', $am->{verify}->($idx));
		agree($node, $am, $t, 'crash in vacuum');
		$node->safe_psql('postgres', "VACUUM $t");
		$node->safe_psql('postgres', $am->{verify}->($idx));
		agree($node, $am, $t, 'vacuum completed after the crash');
		note("$n: $hits kills landed while the operation was running");
	};
}

$node->stop;
done_testing();