#!/bin/sh
# pg_lease failure-model suite (workflow stage 11).
#
# Covers every failure mode in specs/FAILURE-MODEL.md that is testable
# without an external orchestrator:
#   main-server scenarios:  abort, cancel mid-operation, connection loss
#                           (backend termination), stale ownership,
#                           reconnect, expiration, failed renewal/release,
#                           repeated requests
#   throwaway-instance:     clean restart (D1, D2, R1), unclean crash
#                           (D1, D2), lapse during downtime (D3, R2)
#
# Out of scope here (documented, not testable in this environment):
#   primary/replica failover safety (spec §2: out of scope for safety),
#   hostile owners ignoring fencing (spec §5).
#
# Output: one block per scenario — EXPECTED / ACTUAL / RESULT.

set -u

PORT_MAIN=${PGPORT:-5432}
MAINDB=${PGDATABASE:-postgres}
PSQL_MAIN="psql -X -q -v ON_ERROR_STOP=1 -d $MAINDB"
TMPDIR_F=$(mktemp -d /tmp/pg_lease_failure.XXXXXX)
PGCTL=$(pg_config --bindir)/pg_ctl
INITDB=$(pg_config --bindir)/initdb
# initdb/pg_ctl refuse to run as root (e.g. inside Docker test containers);
# when running as root with gosu available, drop to the postgres user.
SUDO=""
if [ "$(id -u)" -eq 0 ] && command -v gosu >/dev/null 2>&1; then
    SUDO="gosu postgres"
    chown postgres "$TMPDIR_F"
fi
TPORT=55432
PASS=0
FAIL=0

# The throwaway instance must load the extension files installed for the
# MAIN SERVER's version; if pg_config (which provides initdb/pg_ctl and the
# share dir) targets a different major, results are silently wrong — fail
# fast instead (found during API-review testing on multi-version hosts).
SRV_NUM=$(psql -X -tA -d "$MAINDB" -c "SHOW server_version_num")
SRV_MAJOR=$(( $SRV_NUM / 10000 ))
CFG_MAJOR=$(pg_config --version | sed -E 's/.*PostgreSQL ([0-9]+).*/\1/')
if [ "$SRV_MAJOR" != "$CFG_MAJOR" ]; then
    echo "failure-model: version mismatch: server=$SRV_MAJOR pg_config=$CFG_MAJOR" >&2
    echo "  run with the server version's bin dir first in PATH," >&2
    echo "  e.g. PATH=/opt/.../postgresql@$SRV_MAJOR/bin:\$PATH $0" >&2
    exit 2
fi

say() { printf '\n=== %s\n' "$1"; }
check() { # name expected actual
    if [ "$2" = "$3" ]; then
        printf '  RESULT:   PASS (%s = %s)\n' "$1" "$3"
        PASS=$((PASS + 1))
    else
        printf '  RESULT:   FAIL (%s: expected %s, got %s)\n' "$1" "$2" "$3"
        FAIL=$((FAIL + 1))
    fi
}
q() { $PSQL_MAIN -tA -c "$1"; }

trap '$SUDO $PGCTL -D "$TMPDIR_F" stop -m immediate >/dev/null 2>&1; rm -rf "$TMPDIR_F"' EXIT

$PSQL_MAIN -c "CREATE EXTENSION IF NOT EXISTS pg_lease" >/dev/null
# clean slate for suite keys
for k in f-abort f-conn f-cancel f-reconnect f-lapse f-repeat; do
    q "DELETE FROM lease.leases WHERE key = '$k'" >/dev/null
done

# ---------------------------------------------------------------------
say "F1 operation transaction aborts (S8, I6)"
# EXPECTED: acquire inside a transaction that aborts leaves no state.
$PSQL_MAIN -c "BEGIN" -c "SELECT lease.acquire('f-abort', 'alice', interval '1h')" \
          -c "ROLLBACK" >/dev/null
actual=$(q "SELECT count(*) FROM lease.leases WHERE key = 'f-abort'")
printf '  EXPECTED: 0 rows (insert rolled back)\n  ACTUAL:   %s rows\n' "$actual"
check F1 0 "$actual"

# ---------------------------------------------------------------------
say "F2 client crash mid-operation: statement canceled (S2 partial ops, I6)"
# EXPECTED: canceling a blocking acquire errors out and changes nothing.
q "SELECT lease.acquire('f-cancel', 'alice', interval '1h')" >/dev/null
q "SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'f-cancel-holder'" >/dev/null 2>&1
( $PSQL_MAIN -c "SET application_name = 'f-cancel-victim';
                 SELECT lease.acquire('f-cancel', 'bob', interval '1h', interval '5s')" \
        >/dev/null 2>&1 ) &
BG=$!
sleep 0.3
q "SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'f-cancel-victim'" >/dev/null
wait $BG 2>/dev/null
actual=$(q "SELECT owner FROM lease.inspect('f-cancel')")
printf '  EXPECTED: alice still holds it (canceled op left no trace)\n  ACTUAL:   owner=%s\n' "$actual"
check F2 alice "$actual"

# ---------------------------------------------------------------------
say "F3 holder connection loss (backend terminated) (S2: lease persists, lapses at TTL)"
# EXPECTED: terminating the holder's backend does NOT free the lease
# (no death-signal release in v0.1); it lapses at expiry.
q "SELECT lease.acquire('f-conn', 'victim', interval '400ms')" >/dev/null
q "SELECT pg_terminate_backend(pid) FROM pg_stat_activity
    WHERE pid <> pg_backend_pid() AND query LIKE '%f-conn%'" >/dev/null 2>&1
immediate=$(q "SELECT held FROM lease.inspect('f-conn')")
sleep 0.6
after=$(q "SELECT held FROM lease.inspect('f-conn')")
printf '  EXPECTED: held=t immediately, held=f after TTL\n  ACTUAL:   immediate=%s after=%s\n' "$immediate" "$after"
check F3-after f "$after"

# ---------------------------------------------------------------------
say "F4 reconnect before lapse with client-held epoch (spec Issue 6, I8)"
# EXPECTED: the new session (reconnect) holding the remembered epoch can
# renew; the epoch is a client credential, the server has no session bond.
q "SELECT lease.acquire('f-reconnect', 'alice', interval '1h')" >/dev/null
ep=$(q "SELECT epoch FROM lease.inspect('f-reconnect')")
st=$(q "SELECT status FROM lease.renew('f-reconnect', 'alice', $ep, interval '1h')")
printf '  EXPECTED: ok (credential works from any session)\n  ACTUAL:   renew=%s (epoch=%s)\n' "$st" "$ep"
check F4 ok "$st"

# ---------------------------------------------------------------------
say "F5 stale ownership: failed renewal and release after lapse (S2, S4)"
# EXPECTED: after lapse, renew and release with the holder's own credentials
# fail lapsed (distinguishable from not_owner); the lapsed holder
# re-acquires with a NEW epoch (takeover of own lapsed lease, OPS-12
# semantics) — never the stale epoch.
q "SELECT lease.acquire('f-lapse', 'alice', interval '400ms')" >/dev/null
sleep 0.6
rn=$(q "SELECT status FROM lease.renew('f-lapse', 'alice', 1, interval '1h')")
rl=$(q "SELECT status FROM lease.release('f-lapse', 'alice', 1)")
old_owner=$(q "SELECT owner FROM lease.inspect('f-lapse')")
re_acq=$(q "SELECT acquired::text || ':' || epoch FROM lease.acquire('f-lapse', 'alice', interval '1h')")
printf '  EXPECTED: renew=lapsed release=lapsed owner=<empty>; re-acquire=true:2\n'
printf '  ACTUAL:   renew=%s release=%s owner=%s re-acquire=%s\n' "$rn" "$rl" "$old_owner" "$re_acq"
check F5-renew lapsed "$rn"
check F5-release lapsed "$rl"
check F5-reacquire "true:2" "$re_acq"

# ---------------------------------------------------------------------
say "F6 repeated requests (S4.1 idempotence under repetition)"
# EXPECTED: N identical acquires by the holder all succeed with the same
# epoch and do not extend expiry beyond the original window.
q "SELECT lease.acquire('f-repeat', 'alice', interval '300ms')" >/dev/null
e1=$(q "SELECT epoch FROM lease.acquire('f-repeat', 'alice', interval '300ms')")
e2=$(q "SELECT epoch FROM lease.acquire('f-repeat', 'alice', interval '300ms')")
e3=$(q "SELECT epoch FROM lease.acquire('f-repeat', 'alice', interval '300ms')")
sleep 0.5
lapsed=$(q "SELECT held FROM lease.inspect('f-repeat')")
printf '  EXPECTED: same epoch across repeats; lease lapses despite repeats\n'
printf '  ACTUAL:   epochs=%s/%s/%s lapsed=%s\n' "$e1" "$e2" "$e3" "$lapsed"
check F6-same-epoch "1:1:1" "$e1:$e2:$e3"
check F6-still-lapses f "$lapsed"

# ---------------------------------------------------------------------
say "F7 clean restart: durable state, epochs survive (D1, D2, R1)"
$SUDO $INITDB -D "$TMPDIR_F" >/dev/null
$SUDO $PGCTL -D "$TMPDIR_F" -o "-p $TPORT -k ${PGHOST:-/tmp} -c fsync=on" -l "$TMPDIR_F/log" start >/dev/null
PSQL_T="psql -X -q -v ON_ERROR_STOP= -p $TPORT -d postgres -tA"
$PSQL_T -c "CREATE EXTENSION pg_lease" >/dev/null
$PSQL_T -c "SELECT lease.acquire('f-restart', 'alice', interval '1h')" >/dev/null
$SUDO $PGCTL -D "$TMPDIR_F" restart -m fast >/dev/null
held=$( { $PSQL_T -c "SELECT held::text || ':' || owner || ':' || epoch FROM lease.inspect('f-restart')"; } )
rn=$($PSQL_T -c "SELECT status FROM lease.renew('f-restart', 'alice', 1, interval '1h')")
printf '  EXPECTED: held=true:alice:1 after restart; renew ok (epoch not regressed)\n'
printf '  ACTUAL:   inspect=%s renew=%s\n' "$held" "$rn"
check F7 "true:alice:1" "$held"
check F7-renew ok "$rn"

# ---------------------------------------------------------------------
say "F8 unclean crash: last committed state stands (D1, D2)"
$PSQL_T -c "SELECT lease.acquire('f-crash', 'bob', interval '1h')" >/dev/null
$SUDO $PGCTL -D "$TMPDIR_F" stop -m immediate >/dev/null
$SUDO $PGCTL -D "$TMPDIR_F" -o "-p $TPORT -k ${PGHOST:-/tmp}" -l "$TMPDIR_F/log" start >/dev/null
sleep 0.5
held=$($PSQL_T -c "SELECT held::text || ':' || owner || ':' || epoch FROM lease.inspect('f-crash')")
printf '  EXPECTED: held=true:bob:1 (WAL-recovered)\n  ACTUAL:   inspect=%s\n' "$held"
check F8 "true:bob:1" "$held"

# ---------------------------------------------------------------------
say "F9 lapse during downtime: downtime never extends ownership (D3, R2)"
$PSQL_T -c "SELECT lease.acquire('f-downtime', 'carol', interval '300ms')" >/dev/null
$SUDO $PGCTL -D "$TMPDIR_F" stop -m immediate >/dev/null
sleep 1
$SUDO $PGCTL -D "$TMPDIR_F" -o "-p $TPORT -k ${PGHOST:-/tmp}" -l "$TMPDIR_F/log" start >/dev/null
sleep 0.5
held=$($PSQL_T -c "SELECT held::text FROM lease.inspect('f-downtime')")
rn=$($PSQL_T -c "SELECT status FROM lease.renew('f-downtime', 'carol', 1, interval '1h')")
tk=$($PSQL_T -c "SELECT acquired::text || ':' || epoch FROM lease.acquire('f-downtime', 'dave', interval '1h')")
printf '  EXPECTED: unowned after downtime; carol renew=lapsed; dave takeover epoch 2\n'
printf '  ACTUAL:   held=%s renew=%s takeover=%s\n' "$held" "$rn" "$tk"
check F9-held "false" "$held"
check F9-renew lapsed "$rn"
check F9-takeover "true:2" "$tk"

$SUDO $PGCTL -D "$TMPDIR_F" stop -m fast >/dev/null

# ---------------------------------------------------------------------
say "SUMMARY"
printf '  PASS: %d  FAIL: %d\n' "$PASS" "$FAIL"
[ "$FAIL" -eq 0 ]