--- provision-promote-W8TQ3MZC.ps1 +++ provision-promote-R2VK8FQN.ps1 @@ -33,7 +33,12 @@ return [int]$value } function Assert([bool]$Condition, [string]$Name) { - if (-not $Condition) { throw $Name } + if (-not $Condition) { + $exception = [InvalidOperationException]::new($Name) + $exception.Data['assertion_name'] = $Name + $exception.Data['source_line'] = (Get-PSCallStack)[1].ScriptLineNumber + throw $exception + } } function Full([string]$Path) { if ($Path.StartsWith('\\?\UNC\', [StringComparison]::OrdinalIgnoreCase)) { $Path = '\\' + $Path.Substring(8) } @@ -61,6 +66,43 @@ } function Read-Json([string]$Path) { return ConvertFrom-Json -InputObject ([IO.File]::ReadAllText($Path, $Utf8)) -AsHashtable -DateKind String +} +function Read-SharedText([string]$Path) { + $file = [IO.FileStream]::new($Path, [IO.FileMode]::Open, [IO.FileAccess]::Read, + ([IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete)) + try { + $reader = [IO.StreamReader]::new($file, $Utf8, $true) + try { return $reader.ReadToEnd() } finally { $reader.Dispose() } + } finally { $file.Dispose() } +} +function Read-PromotionLog([string]$Path, [bool]$RolledAbsentBeforeRun) { + $current = Read-SharedText $Path + if ($RolledAbsentBeforeRun -and [IO.File]::Exists("$Path.1")) { + return (Read-SharedText "$Path.1") + "`n" + $current + } + return $current +} +function Has-OwnedPromotion([string]$Log, [long]$BrainPid, [long]$Generation, [string]$Marker) { + $header = '=== spt brain stderr — generation ' + $Generation + ' — pid ' + $BrainPid + ' ===' + $headers = [regex]::Matches($Log, '(?m)^=== spt (?:brain|broker) stderr — generation \d+ — pid \d+ ===\r?$') + for ($i = 0; $i -lt $headers.Count; $i++) { + if ($headers[$i].Value.TrimEnd("`r") -cne $header) { continue } + $start = $headers[$i].Index + $headers[$i].Length + $end = if ($i + 1 -lt $headers.Count) { $headers[$i + 1].Index } else { $Log.Length } + if ($Log.Substring($start, $end - $start).Contains($Marker)) { return $true } + } + return $false +} +function Get-FailureDiagnostic([Management.Automation.ErrorRecord]$Record) { + $cause = $Record.Exception.GetBaseException() + $name = $null + $line = $Record.InvocationInfo.ScriptLineNumber + if ($cause.Data.Contains('assertion_name')) { + $name = [string]$cause.Data['assertion_name'] + $line = [int]$cause.Data['source_line'] + } + return @{ assertion_name = $name; exception_type = $cause.GetType().FullName; + hresult = $cause.HResult; source_line = $line } } function New-Bytes([string]$Path, [byte[]]$Bytes) { $stream = [IO.FileStream]::new($Path, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::Read) @@ -440,6 +482,9 @@ $Failure = 'OWNED_PROMOTION_READINESS_FAILED' Require-Time 25 + $DaemonLogPath = Join-Path $H 'logs/daemon.stderr.log' + # A backup absent before this owned launch can only enter evidence after this run starts. + $RolledAbsentBeforeRun = -not [IO.File]::Exists("$DaemonLogPath.1") $Run = Start-Native 'promotion-daemon' $E @('daemon', 'run') @{ SPT_HOME = $H; SPT_INSTALL_NO_FIREWALL = '1' } 180 'run' # This launcher is a separate run-scoped process, never a descendant of the apply/readiness step. $readySeconds = Cap 60 20 @@ -473,12 +518,13 @@ do { Require-Time 15 Assert (-not $Run.process.HasExited) 'RUN_EXITED_BEFORE_PROMOTION' - $Log = [IO.File]::ReadAllText($Run.stderr_file, $Utf8) + $Log = Read-PromotionLog $DaemonLogPath $RolledAbsentBeforeRun Assert ($Log -notmatch 'PROMOTE_BYTES_UNVERIFIED|BRAIN_PROMOTE_REJECTED|BRAIN_ROLLBACK') 'PROMOTION_BYTES_UNVERIFIED_OR_ROLLBACK' $Applied = Read-Json $StatePath Assert ($Applied.phase -cin @('applied-pending', 'applied')) 'PROMOTION_STATE_REFUSED' $marker = 'BRAIN_PROMOTED: candidate v' + $N + ' signalled ready (generation ' + $BrainReady.generation + ')' - if ($Applied.phase -ceq 'applied' -and (Numeric-N $Applied.version) -and $Log.Contains($marker)) { break } + if ($Applied.phase -ceq 'applied' -and (Numeric-N $Applied.version) -and + (Has-OwnedPromotion $Log $Membership.brain.pid $BrainReady.generation $marker)) { break } Assert ((Now) -lt $PromotionEnd) 'DURABLE_PROMOTION_WAIT_EXHAUSTED' Start-Sleep -Milliseconds 100 } while ($true) @@ -590,6 +636,7 @@ Write-Output 'PROMOTION_COMPLETE=YES' } catch { + $FailureDiagnostic = Get-FailureDiagnostic $_ # Never print exception messages, environment values or unqualified path/error payloads. # On any failure the same owned run stop channel is the sole shutdown request. if ($null -ne $Run) { @@ -607,11 +654,13 @@ } if ($OwnOutput) { try { New-Json (Join-Path $OutputRoot 'promotion-failure.json') @{ accepted = $false; error = $Failure; + failure = $FailureDiagnostic; shutdown = $Shutdown; deadline = $Deadline; recorded_utc = [DateTime]::UtcNow.ToString('o'); native_records = @($Owned | ForEach-Object { $_.record_file }); public_partials_preserved = $true; rig_disposal_performed = $false; product_applied_state_written_by_instrument = $false } } catch { } } [Console]::Error.WriteLine($Failure) + [Console]::Error.WriteLine((ConvertTo-Json -Compress -InputObject $FailureDiagnostic)) exit 1 } finally {