Files
seaweedfs/weed/storage/blockvol/testrunner/scripts/run-phase20-t6.ps1
T
pingqiuandClaude Opus 4.6 55013e103b feat: Phase 20 Stage 0+1 closure — bootstrap + sustained workload on hardware
Stage 0 (bootstrap closure): PASS on m01/M02
  - create RF=2 sync_all → 10s shipper wait → 4k fsync → publish_healthy
  - Proves: BarrierAccepted observation, ShipperConnected, DurableLSN > 0

Stage 1 (sustained workload): 32/33 actions PASS
  - bootstrap → fio 10s randwrite → dd_write 1M×2 fsync → data checksum
  - Remaining: auto-failover promotion (separate issue)

Key fixes:
  - BarrierAccepted callback: SyncCache success → core DurableLSN update
  - BarrierRejected callback: barrier failures surface to core with reason
  - Shipper state callback for new volumes (not just startup volumes)
  - CatchUpTo ctrl conn reset: prevents stale control channel after recovery
  - CP13-6 max-bytes budget suspended: uses replicaFlushedLSN which can't
    advance without barrier; kills healthy shippers during async writes.
    Will be replaced by v2 negotiated sync/recovery protocol.
  - Barrier diagnostic logging: start/fail/success with reason and LSN
  - Scenario restructured: Stage 0 (bootstrap-closure) + Stage 1 (failover)
  - dd_write: sync_mode param + real stderr capture
  - sw-test-runner suite command: deploy once, run N scenarios
  - WAL size plumbing: proto + API + handler (forward-compatible)

Known: 6 blockvol/server test failures from Barrier() path change
(bounded catch-up in Barrier). Need test updates to match new semantics.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-06 19:55:12 -07:00

162 lines
7.1 KiB
PowerShell

param(
[ValidateSet("list", "stage0", "stage1", "stage2-readiness", "stage3-compare")]
[string]$Stage = "list",
[string]$RunnerPath = "sw-test-runner",
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..\..\..\..")).Path,
[string]$ResultsDir = "results/phase20-t6",
[switch]$DryRun
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
function New-Scenario {
param(
[string]$Id,
[string]$Path,
[string]$Purpose
)
return [pscustomobject]@{
Id = $Id
Path = $Path
Purpose = $Purpose
}
}
function Write-Pack {
param(
[string]$Title,
[object[]]$Scenarios
)
Write-Host ""
Write-Host $Title
Write-Host ("-" * $Title.Length)
foreach ($scenario in $Scenarios) {
Write-Host ("{0}: {1}" -f $scenario.Id, $scenario.Path)
Write-Host (" purpose: {0}" -f $scenario.Purpose)
}
}
function Invoke-Pack {
param(
[string]$Title,
[object[]]$Scenarios
)
Write-Pack -Title $Title -Scenarios $Scenarios
Write-Host ""
Write-Host ("Runner: {0}" -f $RunnerPath)
Write-Host ("RepoRoot: {0}" -f $RepoRoot)
Write-Host ("ResultsDir: {0}" -f $ResultsDir)
foreach ($scenario in $Scenarios) {
$scenarioPath = Join-Path $RepoRoot $scenario.Path
if (-not (Test-Path $scenarioPath)) {
throw ("Scenario not found: {0}" -f $scenarioPath)
}
$runResultsDir = Join-Path $RepoRoot (Join-Path $ResultsDir $scenario.Id)
$cmd = @(
$RunnerPath,
"run",
$scenarioPath,
"--results-dir",
$runResultsDir
)
Write-Host ""
Write-Host ("[{0}] {1}" -f $scenario.Id, $scenario.Purpose)
Write-Host ("CMD: {0}" -f ($cmd -join " "))
if (-not $DryRun) {
& $RunnerPath run $scenarioPath --results-dir $runResultsDir
if ($LASTEXITCODE -ne 0) {
throw ("Scenario failed: {0}" -f $scenario.Id)
}
}
}
}
$stage0Pack = @(
(New-Scenario -Id "P20-H0" -Path "weed/storage/blockvol/testrunner/scenarios/internal/recovery-bootstrap-closure.yaml" -Purpose "Stage 0 bootstrap closure: create -> first fsync fence -> publish_healthy on the healthy RF=2 sync_all path")
)
$stage1Pack = @(
(New-Scenario -Id "P20-T6-H1A" -Path "weed/storage/blockvol/testrunner/scenarios/internal/recovery-baseline-failover.yaml" -Purpose "V1 auto-failover baseline with data continuity"),
(New-Scenario -Id "P20-T6-H1B" -Path "weed/storage/blockvol/testrunner/scenarios/internal/suite-ha-failover.yaml" -Purpose "HA failover with cluster health checks"),
(New-Scenario -Id "P20-T6-H1C" -Path "weed/storage/blockvol/testrunner/scenarios/cp11b3-manual-promote.yaml" -Purpose "Manual promote plus preflight and rejoin recovery"),
(New-Scenario -Id "P20-T6-H1D" -Path "weed/storage/blockvol/testrunner/scenarios/lease-expiry-write-gate.yaml" -Purpose "Lease/write gate regression while T6 work proceeds")
)
$stage2ReadinessPack = @(
(New-Scenario -Id "P20-T6-H2" -Path "weed/storage/blockvol/testrunner/scenarios/internal/recovery-baseline-failover.yaml" -Purpose "Future Stage 2 copy with --block.v2Promotion=true for durability-first failover"),
(New-Scenario -Id "P20-T6-H3" -Path "weed/storage/blockvol/testrunner/scenarios/internal/recovery-baseline-failover.yaml" -Purpose "Future ambiguous-evidence variant with fail-closed expectation"),
(New-Scenario -Id "P20-T6-H4" -Path "weed/storage/blockvol/testrunner/scenarios/internal/v2-failover-gate.yaml" -Purpose "Future gated-promotion scenario; expected to be added during T6/T7")
)
switch ($Stage) {
"list" {
Write-Pack -Title "Stage 0 Pack (Bootstrap Closure First)" -Scenarios $stage0Pack
Write-Pack -Title "Stage 1 Pack (Runnable Today)" -Scenarios $stage1Pack
Write-Pack -Title "Stage 2 Readiness Pack (Do Not Run Until Ready)" -Scenarios $stage2ReadinessPack
Write-Host ""
Write-Host "Notes:"
Write-Host " - Stage 0 is the first hardware gate: close P20-H0 / P20-A2 / P20-A5 before stronger V2 failover claims."
Write-Host " - Current master flag spelling is --block.v2Promotion."
Write-Host " - Stage 1 uses existing YAMLs because V1 failover is the default path."
Write-Host " - Stage 2 requires dedicated YAML copies or overlays with start_weed_master.extra_args += -block.v2Promotion=true."
}
"stage0" {
Write-Host "Stage 0 runs the bootstrap closure baseline before any stronger T6/T7 claims."
Write-Host "It is the hardware entrypoint for P20-H0 plus acceptance overlay P20-A2/P20-A5."
Write-Host ""
Write-Host "Record during each run:"
Write-Host " 1. /debug/block/shipper on both candidate volume servers"
Write-Host " 2. /block/volume/<name> on master before failover, immediately after failover, and after recovery window"
Write-Host " 3. whether ReplicaIDs, ShipperConfigured, and publish_healthy converge together"
Write-Host ""
Write-Host "Pass requires:"
Write-Host " 1. promoted primary no longer shows ReplicaIDs=[] on the healthy path"
Write-Host " 2. ShipperConfigured=true can be reached on the promoted primary"
Write-Host " 3. publish_healthy=true can be reached without transport-contact shortcuts"
Invoke-Pack -Title "Stage 0 Pack" -Scenarios $stage0Pack
}
"stage1" {
Write-Host "Stage 1 runs existing YAMLs with V1 failover authority and V2 observation surfaces."
Write-Host "Use --block.v2Promotion=false semantics; existing YAMLs can remain unchanged."
Invoke-Pack -Title "Stage 1 Pack" -Scenarios $stage1Pack
}
"stage2-readiness" {
Write-Pack -Title "Stage 2 Readiness Pack" -Scenarios $stage2ReadinessPack
Write-Host ""
Write-Host "Do not run Stage 2 until all of the following are true:"
Write-Host " 1. ReplicaIDs is non-empty on the healthy promoted primary path."
Write-Host " 2. ShipperConfigured=true and publish_healthy=true can be reached on hardware."
Write-Host " 3. Evidence transport is real, not placeholder-only."
Write-Host " 4. Scenario copies explicitly set --block.v2Promotion=true on the master."
Write-Host ""
Write-Host "This wrapper intentionally does not rewrite YAML files."
}
"stage3-compare" {
Write-Host "Stage 3 compare pack:"
Write-Host " 1. Run Stage 1 baseline scenarios under V1 failover."
Write-Host " 2. Re-run equivalent Stage 2 copies under V2 failover."
Write-Host " 3. Compare selected primary, epoch, data continuity, engine_projection_mode, cluster_replication_mode, and gate behavior."
Write-Host ""
Write-Host "Use:"
Write-Host " .\\weed\\storage\\blockvol\\testrunner\\scripts\\run-phase20-t6.ps1 -Stage stage0"
Write-Host " .\\weed\\storage\\blockvol\\testrunner\\scripts\\run-phase20-t6.ps1 -Stage stage1"
Write-Host " .\\weed\\storage\\blockvol\\testrunner\\scripts\\run-phase20-t6.ps1 -Stage stage2-readiness"
}
}