#!/usr/bin/env bash
# Process census + scoped post-job reap of test-spawned daemons (Linux CI leg).
#
# The Linux analog of reap-census.ps1. Same contract, and it is here for SYMMETRY OF
# EVIDENCE rather than because kitsubito has the Windows box's problem: kitsubito does not
# host the live agent fleet, so its leaks do not pick a random victim out of a saturated
# process table. What it DOES have is the flood-orphan class the pre-checkout guard already
# names — a `yes FLOOD…` child reparented to init after a nextest slow-timeout SIGKILL,
# burning a full core forever — plus workspace daemons that survive a killed battery. Both
# contend with the NEXT run, and both are invisible today because nothing counts them.
#
# MEMBERSHIP IS DECIDED BY EXE BASENAME OFF A DIRECT /proc WALK, not by asking pgrep to
# pre-filter on comm. Three reasons, all of them defects this script shipped with in gate
# round 1 (doyle, confirmed on kitsubito itself, procps-ng 4.0.4):
#
#   1. `pgrep -E` DOES NOT EXIST in procps ("invalid option -- 'E'", exit 2). pgrep patterns
#      are ERE already. Combined with `2>/dev/null || true` that made the whole Linux leg a
#      SILENT NO-OP: family_total=0 forever, reap loop never iterating, and a log that reads
#      EXACTLY like a clean box.
#   2. comm IS TRUNCATED TO 15 CHARS by the kernel, and three of our family names are longer
#      (translate_proof_fixture=23, post_step_fixture=17, dispatch_fixture=16). pgrep matches
#      /proc/<pid>/status Name, so those three could never match — procps even warns about it
#      on stderr, and that warning was eaten too.
#   3. The obvious patch — `pgrep -f` — is WORSE, not better: full-cmdline substring matching
#      is precisely the machine-wide-match class this script's own kill-scope rules forbid.
#
# Walking /proc and classifying on the exe path makes that path the SINGLE SOURCE OF TRUTH
# for both membership and scope, and the 15-char limit stops existing. comm is consulted
# only as a fallback for a process whose exe link is unreadable, purely so it stays VISIBLE
# in the census — such a process is never a kill candidate.
#
# AND, THE STANDING LESSON THIS FILE IS THE TEXTBOOK CASE OF: a matcher that errors into
# empty output is indistinguishable from a clean box, so ABSENCE NEEDS A SIBLING PROBE. The
# self-check below proves the enumerator can see SOMETHING before any census is believed.
#
# KILL SCOPE IS PATH-VERIFIED AND NARROW, re-resolved per-pid at kill time via
# /proc/<pid>/exe — never a bare `pkill spt`. Live infra under a `spt-core/bin/` install
# prefix and any `owl` binary are hard-excluded even if a root is ever computed too wide,
# and a process whose exe link cannot be read is reported but never killed: unreadable
# means unverifiable, and the safe direction is to leave it alone.
#
# [impl->REQ-CI-POSTJOB-DAEMON-REAP]
set -uo pipefail

# Sourced with SPT_CI_REAP_LIB=1, this file DEFINES its predicates and does nothing else —
# see reap-census-selftest.sh. The classification logic is pure string work, so it can be
# asserted on any platform, which matters because the three defects this script shipped with
# in gate round 1 were ALL in that logic and ALL invisible without a Linux box to run on.
# A rig that only exists on the machine that already has the problem is not a rig.
if [ "${SPT_CI_REAP_LIB:-0}" != 1 ]; then
  PHASE="${1:?usage: reap-census.sh <start|attribution-control|end>}"
fi
SETTLE_SECS="${SPT_CI_REAP_SETTLE_SECS:-5}"

# The exe basenames this workspace's tests spawn. `owl` is listed for CENSUS VISIBILITY
# only — it is hard-excluded from reaping below. Space-delimited so membership is an exact
# whole-word test, never a substring one.
FAMILY_SET=" spt owl notify-shell mock-session mock-shell dispatch_fixture translate_proof_fixture post_step_fixture gh_fixture git_fixture "

# Every directory tree whose binaries THIS run is responsible for. A daemon running from
# outside all of them belongs to somebody else.
roots=()
for r in "${GITHUB_WORKSPACE:-}" "${CARGO_TARGET_DIR:-}" \
         "${GITHUB_WORKSPACE:+$GITHUB_WORKSPACE/target}" \
         "${GITHUB_WORKSPACE:+$GITHUB_WORKSPACE/.adapter-notify/target}" \
         "${RUNNER_TEMP:-}" "${HOME:+$HOME/spt-n1-oldbroker}"; do
  [ -n "$r" ] || continue
  roots+=("${r%/}")
done

is_family() {
  case "$FAMILY_SET" in *" $1 "*) return 0 ;; esac
  return 1
}

# A rebuild can unlink a binary out from under a still-running daemon — ordinary inside a
# CI battery, not a corner case — and the kernel then reports the link as "<path> (deleted)".
# Such a process is still ours and still a leak, so the suffix has to come off before either
# the membership test or the scope test sees the path.
#
# THE STRIP IS UNCONDITIONAL, and round 2 is why. The first version applied it only on a
# fallback path, on the premise that `readlink -f` FAILS on a deleted exe. That premise is
# false on a real box: -f only requires the leading components to exist, so it SUCCEEDS and
# returns the path WITH the suffix still attached. The fallback was therefore never reached,
# the strip never ran, and the basename came out as "mock-session (deleted)" — which matches
# no family name, so the process was absent from every census and survived the reap. The
# exact leak class the handling was added for sailed straight through it (doyle, kitsubito,
# gate round 2, demonstrated with a spawned-then-unlinked decoy).
strip_deleted() { printf '%s' "${1% (deleted)}"; }

# Pure: raw readlink output in, the basename to test for family membership out. Split out as
# its own function precisely because the RUNTIME behaviour above needed a Linux box to
# discover but this STRING TRANSFORM does not — which is this script's own rig thesis, and
# round 2 caught me not applying it to the code I had just added.
exe_family_name() {
  local p
  p=$(strip_deleted "$1")
  printf '%s' "${p##*/}"
}

# The exe path behind a pid (suffix stripped), or empty when it cannot be resolved.
exe_of() {
  local e
  e=$(readlink -f "/proc/$1/exe" 2>/dev/null) || e=""
  [ -n "$e" ] || e=$(readlink "/proc/$1/exe" 2>/dev/null) || e=""
  strip_deleted "$e"
}

# comm, for VISIBILITY ONLY when the exe link is unreadable. Kernel-truncated to 15 chars,
# which is exactly why it is not trusted for membership — see the header.
comm_of() { cat "/proc/$1/comm" 2>/dev/null | tr -d '\n'; }

# The nextest process-per-test runner exports this identity into each test
# process; daemon and brain descendants inherit it, and retain it after PPID
# becomes 1. Reading the survivor's own environment therefore attributes a leak
# without guessing from timing or from whichever test happened to fail nearby.
# [impl->REQ-CI-CENSUS-TEST-ATTRIBUTION]
nextest_test_of() {
  local value
  value=$(tr '\0' '\n' <"/proc/$1/environ" 2>/dev/null |
    sed -n 's/^NEXTEST_TEST_NAME=//p' | sed -n '1p')
  [ -n "$value" ] || value='<not-nextest-or-unreadable>'
  printf '%s' "$value" | tr '[:space:]' '_'
}

comm_is_family() {
  local c="$1" name
  [ -n "$c" ] || return 1
  for name in $FAMILY_SET; do
    [ "$c" = "${name:0:15}" ] && return 0
  done
  return 1
}

# Hard exclusions, applied AFTER the root test rather than instead of it: if a root is ever
# computed too wide, this is what still stands between the reap and production.
#
# `*/.local/bin/*` is the LINUX install layout and is listed because the Windows-shaped
# `*/spt-core/bin/*` guards nothing on this box: kitsubito's live fleet binary is
# ~/.local/bin/spt (confirmed from the unit's MainPID exe), which that pattern never matches.
# A backstop that cannot match the thing it exists to protect is decoration. Broad is safe
# here — CI never builds into .local/bin (doyle ruling, gate round 2).
is_excluded() {
  case "$1" in
    */spt-core/bin/*) return 0 ;;
    */.local/bin/*)   return 0 ;;
    */owl)            return 0 ;;
  esac
  return 1
}

in_scope() {
  local exe="$1"
  [ -n "$exe" ] || return 1          # unreadable => never eligible
  is_excluded "$exe" && return 1
  local r
  for r in "${roots[@]}"; do
    case "$exe" in "$r"/*) return 0 ;; esac
  done
  return 1
}

# ABSENCE NEEDS A SIBLING PROBE. A census of zero must mean "the box is clean", never "the
# enumerator is broken" — those two produced identical logs in gate round 1, which is how a
# no-op shipped. Prove the rig works before any number from it is believed.
self_check() {
  local seen=0 d
  for d in /proc/[0-9]*; do
    [ -d "$d" ] && seen=$((seen + 1))
  done
  if [ "$seen" -lt 2 ]; then
    echo "::error::CI-CENSUS self-check FAILED: walked $seen entries under /proc. The enumerator is broken, and a broken enumerator reports family_total=0 — indistinguishable from a clean box. Treat every census below as UNKNOWN, not as zero."
    return 1
  fi
  if [ -z "$(exe_of self)" ]; then
    echo "::error::CI-CENSUS self-check FAILED: cannot resolve /proc/self/exe. The exe path is the membership AND scope oracle here, so every row would silently read UNREAD and nothing would ever be eligible for reaping."
    return 1
  fi
  echo "CI-CENSUS self-check OK: walked $seen /proc entries, exe resolution live"
  return 0
}

# Flood orphans are `yes FLOOD…` — a unique marker that cannot match an unrelated process,
# and they are reparented to init so no path scoping is possible or needed. This is the ONE
# sanctioned full-cmdline match (the pre-checkout guard already uses it), and it is kept out
# of family membership entirely.
#
# Counted with `pgrep | wc -l` rather than `pgrep -c`: on no-match, pgrep -c PRINTS "0" and
# THEN exits 1, so a `|| echo 0` fallback fires AFTER the print and captures "0\n0" — which
# lands a newline inside the summary line and breaks the one-greppable-line contract the
# census exists for. An exit above 1 is a usage error, not a no-match, and is reported as
# UNKNOWN rather than silently folded into zero.
flood_count() {
  local out rc
  out=$(pgrep -f 'yes FLOOD' 2>/dev/null)
  rc=$?
  if [ "$rc" -gt 1 ]; then
    echo "::warning::CI-CENSUS: the flood-orphan probe failed (pgrep exit $rc — a usage error, not a no-match). Flood count is UNKNOWN, not zero." >&2
    printf 'unknown'
    return
  fi
  printf '%s' "$(printf '%s' "$out" | grep -c '[0-9]')"
}

# Publishes the scoped count in CENSUS_SCOPED; human-readable rows go to stdout under one
# greppable summary line, so a run-to-run diff is a grep rather than a read.
census() {
  local label="$1" scoped=0 total=0 unreadable=0 pid exe tag test_name rows="" d c
  local box_procs=0
  # file-nr's first field is system-wide allocated file handles — the cheap analog of the
  # Windows census's box-wide handle count, with no lsof sweep.
  local box_fds; box_fds=$(awk '{print $1}' /proc/sys/fs/file-nr 2>/dev/null || echo 0)

  for d in /proc/[0-9]*; do
    [ -d "$d" ] || continue
    box_procs=$((box_procs + 1))
    pid=${d#/proc/}
    exe=$(exe_of "$pid")
    if [ -n "$exe" ]; then
      is_family "$(exe_family_name "$exe")" || continue
    else
      # Unreadable exe: fall back to comm PURELY so the process stays visible. Never a kill
      # candidate — in_scope() rejects an empty exe by construction.
      c=$(comm_of "$pid")
      comm_is_family "$c" || continue
    fi
    total=$((total + 1))
    if in_scope "$exe"; then tag='SCOPED  '; scoped=$((scoped + 1))
    elif [ -z "$exe" ];  then tag='UNREAD  '; unreadable=$((unreadable + 1))
    else                      tag='FOREIGN '
    fi
    test_name=$(nextest_test_of "$pid")
    rows+="  $tag pid=$pid test=$test_name ${exe:-<exe link unreadable — not ours to kill> comm=${c:-?}}"$'\n'
  done

  local floods; floods=$(flood_count)
  echo "CI-CENSUS phase=$label box_procs=$box_procs box_fds=$box_fds family_total=$total scoped=$scoped unscoped=$((total - scoped)) unreadable_path=$unreadable flood_orphans=$floods"
  printf '%s' "$rows"
  CENSUS_SCOPED=$scoped
  CENSUS_UNREADABLE=$unreadable
}

# A numerical zero is a verdict only when the enumerator saw every family path.
# Keep this pure so the self-test can mutation-pin the unreadable population.
survivor_verdict() {
  local observed="$1" unreadable="$2"
  if [ "$unreadable" -gt 0 ]; then printf 'UNPROVEN'; else printf '%s' "$observed"; fi
}

# Positive control for per-test attribution. It drives the same `spt daemon run`
# spawn/re-exec path as the leaking E2Es, then reads the inherited nextest name
# from BOTH live process classes before any missing attribution is interpreted.
# [impl->REQ-CI-CENSUS-TEST-ATTRIBUTION]
attribution_control() {
  local target="${CARGO_TARGET_DIR:-${GITHUB_WORKSPACE:-}/target}"
  local spt_bin="${target%/}/debug/spt"
  if [ ! -x "$spt_bin" ]; then
    echo "::error::CI-CENSUS attribution control: built spt not found at $spt_bin"
    return 1
  fi
  local control_home
  control_home=$(mktemp -d "${RUNNER_TEMP:-/tmp}/spt-census-control.XXXXXX") || return 1
  mkdir -p "$control_home/identity"
  printf 'not-a-valid-seed' >"$control_home/identity/node.key"
  env -u OWL_SESSION_ID -u SPT_AGENT_ID -u SPT_ENDPOINT_ID \
    SPT_HOME="$control_home" NEXTEST_TEST_NAME='CI-CENSUS::positive-control' \
    "$spt_bin" daemon run >"$control_home/stdout.log" 2>"$control_home/stderr.log" &
  local broker_pid=$! brain_pid='' broker_name='' brain_name='' i
  for i in $(seq 1 600); do
    brain_pid=$(sed -n 's/.*"pid"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' \
      "$control_home/brain.ready" 2>/dev/null | sed -n '1p')
    [ -n "$brain_pid" ] && break
    sleep 0.1
  done
  broker_name=$(nextest_test_of "$broker_pid")
  [ -n "$brain_pid" ] && brain_name=$(nextest_test_of "$brain_pid")

  timeout 15s env -u OWL_SESSION_ID -u SPT_AGENT_ID -u SPT_ENDPOINT_ID \
    SPT_HOME="$control_home" "$spt_bin" daemon stop --force \
    >"$control_home/stop.log" 2>&1 || true
  for i in $(seq 1 50); do
    if ! kill -0 "$broker_pid" 2>/dev/null &&
       { [ -z "$brain_pid" ] || ! kill -0 "$brain_pid" 2>/dev/null; }; then
      break
    fi
    sleep 0.1
  done
  if kill -0 "$broker_pid" 2>/dev/null ||
     { [ -n "$brain_pid" ] && kill -0 "$brain_pid" 2>/dev/null; }; then
    echo "::error::CI-CENSUS attribution control cleanup unproven; preserving $control_home (broker=$broker_pid brain=${brain_pid:-unknown})"
    return 1
  fi
  wait "$broker_pid" 2>/dev/null || true
  rm -rf "$control_home"

  if [ "$broker_name" != 'CI-CENSUS::positive-control' ] ||
     [ "$brain_name" != 'CI-CENSUS::positive-control' ]; then
    echo "::error::CI-CENSUS attribution control FAILED: broker=$broker_name brain=${brain_name:-<not-started>}. Missing survivor attribution is UNKNOWN; name no test."
    return 1
  fi
  echo "CI-CENSUS attribution control OK: broker and brain retained NEXTEST_TEST_NAME across the production daemon spawn path"
}

# ---- Predicates end here. Sourced as a library, we stop before touching the box. ----
if [ "${SPT_CI_REAP_LIB:-0}" = 1 ]; then
  return 0 2>/dev/null || exit 0
fi

echo "CI-CENSUS scope roots:"
for r in "${roots[@]}"; do echo "  root: $r"; done

# In WARN mode a failed self-check annotates loudly and the run continues, which is right:
# a broken enumerator should not red an otherwise-green battery while this leg is burning in.
# RIDER FOR THE STRICT-FLIP COMMIT (doyle, gate round 2 — recorded here so it lands with the
# flip and not after it): once SPT_CI_REAP_STRICT is the default, a FAILED self-check must
# ALSO exit 1. Under strict, "the enumerator is broken" must never be reportable as
# scoped_survivors=0, which is exactly what today's continue-and-exit-0 path would do.
self_check || true

if [ "$PHASE" = attribution-control ]; then
  attribution_control
  exit $?
fi

if [ "$PHASE" = start ]; then
  census start
  if [ "${CENSUS_SCOPED:-0}" -gt 0 ]; then
    echo "::warning::CI-CENSUS start: ${CENSUS_SCOPED} scoped daemon(s) inherited from a previous run — this battery is starting contaminated."
  fi
  exit 0
fi

# ---- Phase: end. Settle, reap, then census. ----
census pre-reap

# BOUNDED SETTLE BEFORE THE KILL PASS (doyle ruling 2026-07-22): a daemon exiting cleanly on
# its own is not a leak, and counting it as one would make the survivor number mean two
# different things. After this, anything still alive is a leak by definition.
if [ "${CENSUS_SCOPED:-0}" -gt 0 ]; then
  echo "CI-REAP: settling ${SETTLE_SECS}s so cleanly-exiting daemons are not counted as leaks"
  sleep "$SETTLE_SECS"
  census post-settle
fi

killed=0
kill_failed=0
for d in /proc/[0-9]*; do
  [ -d "$d" ] || continue
  pid=${d#/proc/}
  # RE-RESOLVE AT KILL TIME. The census above is a moment in the past and a pid can be
  # recycled between then and now into anything at all.
  exe=$(exe_of "$pid")
  [ -n "$exe" ] || continue
  is_family "$(exe_family_name "$exe")" || continue
  in_scope "$exe" || continue
  if kill -9 "$pid" 2>/dev/null; then
    killed=$((killed + 1))
    echo "  REAP  pid=$pid $exe"
  else
    kill_failed=$((kill_failed + 1))
    echo "::warning::CI-REAP could not stop pid=$pid $exe"
  fi
done

if pkill -9 -f 'yes FLOOD' 2>/dev/null; then floods_reaped=reaped; else floods_reaped=none; fi

sleep 1
census end
unreadable=${CENSUS_UNREADABLE:-0}
observed=${CENSUS_SCOPED:-0}
verdict=$(survivor_verdict "$observed" "$unreadable")
echo "CI-REAP summary: killed=$killed kill_failed=$kill_failed flood_orphans=$floods_reaped scoped_survivors=$verdict observed_scoped=$observed unreadable_path=$unreadable"

if [ "$observed" -gt 0 ] || [ "$unreadable" -gt 0 ]; then
  if [ "$unreadable" -gt 0 ]; then
    msg="CI-REAP: clean survivor count REFUSED — $unreadable family process path(s) were unreadable, beside $observed observed scoped survivor(s)."
  else
    msg="CI-REAP: $observed test-spawned daemon(s) survived the post-job reap — the next run on this runner starts contaminated."
  fi
  if [ "${SPT_CI_REAP_STRICT:-0}" = 1 ]; then
    echo "::error::$msg"
    exit 1
  fi
  echo "::warning::$msg"
fi
exit 0