Files
seaweedfs/weed/storage/blockvol/engine_metrics.go
T
Ping QiuandClaude Opus 4.6 da1b81d1c9 feat: CP8-3-1 durability modes + testrunner platform + 21 adversarial tests
Durability mode implementation (sync_all, sync_quorum, best_effort):
- DurabilityMode type with superblock persistence, parse/validate/string
- MakeDistributedSync mode-aware barrier enforcement in dist_group_commit
- blockerr sentinel package (ErrDurabilityBarrierFailed, ErrDurabilityQuorumLost)
- gRPC create path: mode validation, idempotent create consistency, partial cleanup
- F1: strict mode rejects partial replica provisioning with cleanup
- F3: empty heartbeat does not overwrite persisted strict mode
- F4: SCSI error mapping uses errors.Is sentinels (not string matching)
- Proto/wire/blockapi/CLI/UI plumbing for durability_mode field
- Observability dashboard: cluster health cards + per-volume columns

Testrunner platform (YAML-driven integration test framework):
- Engine, parser, registry, reporter (JUnit XML + HTML), metrics scraping
- 52 registered actions: block, iSCSI, I/O, fault injection, assertions
- Baseline regression framework with 7 hard-fail conditions
- 15 YAML scenarios (smoke, crash, HA, fault, consistency, snapshot)
- 49 unit tests for testrunner internals

QA adversarial suite (21 tests, all PASS):
- Idempotent create mode/RF mismatch detection
- Heartbeat mode downgrade prevention (F3)
- sync_all/sync_quorum partial replica enforcement (F1)
- Concurrent create race safety
- Failover/expand mode preservation
- Cleanup resilience when delete fails
- Master restart auto-register mode handling
- Superblock roundtrip all 3 modes
- Validate edge cases (mode×RF matrix)
- RequiredReplicas quorum math verification
- Sentinel error categorization

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 01:06:51 -08:00

124 lines
3.8 KiB
Go

package blockvol
import (
"sync/atomic"
"time"
)
// EngineMetrics collects subsystem-level counters for Prometheus export.
// All fields are safe for concurrent access.
type EngineMetrics struct {
// Flusher
FlusherBytesTotal atomic.Uint64
FlusherFlushesTotal atomic.Uint64
FlusherErrorsTotal atomic.Uint64
flusherLatencyNs atomicHistogram // flush cycle duration
// Group Commit
GroupCommitBatchTotal atomic.Uint64 // alias for GroupCommitter.syncCount
groupCommitBatchSize atomicHistogram
groupCommitWaitNs atomicHistogram
// WAL Shipper
WALShippedEntriesTotal atomic.Uint64
WALBarrierRequestsTotal atomic.Uint64
WALFailedBarriersTotal atomic.Uint64
walBarrierLatencyNs atomicHistogram
// Scrub
ScrubPassesTotal atomic.Uint64
ScrubErrorsTotal atomic.Uint64
scrubDurationNs atomicHistogram
// Durability (CP8-3-1)
DurabilityBarrierFailedTotal atomic.Uint64 // sync_all barrier failures
DurabilityQuorumLostTotal atomic.Uint64 // sync_quorum quorum lost
}
// NewEngineMetrics creates an EngineMetrics instance.
func NewEngineMetrics() *EngineMetrics {
return &EngineMetrics{}
}
// RecordFlusherFlush records a completed flush cycle.
func (m *EngineMetrics) RecordFlusherFlush(bytesWritten uint64, dur time.Duration) {
m.FlusherBytesTotal.Add(bytesWritten)
m.FlusherFlushesTotal.Add(1)
m.flusherLatencyNs.record(dur.Nanoseconds())
}
// RecordFlusherError records a flush error.
func (m *EngineMetrics) RecordFlusherError() {
m.FlusherErrorsTotal.Add(1)
}
// RecordGroupCommitBatch records a group commit batch.
func (m *EngineMetrics) RecordGroupCommitBatch(batchSize int, waitDur time.Duration) {
m.GroupCommitBatchTotal.Add(1)
m.groupCommitBatchSize.record(int64(batchSize))
m.groupCommitWaitNs.record(waitDur.Nanoseconds())
}
// RecordWALShipped records a shipped WAL entry.
func (m *EngineMetrics) RecordWALShipped() {
m.WALShippedEntriesTotal.Add(1)
}
// RecordWALBarrier records a barrier request with its result.
func (m *EngineMetrics) RecordWALBarrier(dur time.Duration, failed bool) {
m.WALBarrierRequestsTotal.Add(1)
m.walBarrierLatencyNs.record(dur.Nanoseconds())
if failed {
m.WALFailedBarriersTotal.Add(1)
}
}
// RecordScrubPass records a completed scrub pass.
func (m *EngineMetrics) RecordScrubPass(dur time.Duration, errors int64) {
m.ScrubPassesTotal.Add(1)
m.ScrubErrorsTotal.Add(uint64(errors))
m.scrubDurationNs.record(dur.Nanoseconds())
}
// FlusherLatencySnapshot returns flusher latency stats (count, sum_ns).
func (m *EngineMetrics) FlusherLatencySnapshot() (count uint64, sumNs int64) {
return m.flusherLatencyNs.snapshot()
}
// GroupCommitBatchSizeSnapshot returns batch size stats.
func (m *EngineMetrics) GroupCommitBatchSizeSnapshot() (count uint64, sum int64) {
return m.groupCommitBatchSize.snapshot()
}
// GroupCommitWaitSnapshot returns group commit wait time stats.
func (m *EngineMetrics) GroupCommitWaitSnapshot() (count uint64, sumNs int64) {
return m.groupCommitWaitNs.snapshot()
}
// WALBarrierLatencySnapshot returns barrier latency stats.
func (m *EngineMetrics) WALBarrierLatencySnapshot() (count uint64, sumNs int64) {
return m.walBarrierLatencyNs.snapshot()
}
// ScrubDurationSnapshot returns scrub duration stats.
func (m *EngineMetrics) ScrubDurationSnapshot() (count uint64, sumNs int64) {
return m.scrubDurationNs.snapshot()
}
// atomicHistogram is a minimal lock-free histogram that tracks count and sum.
// Prometheus histograms are registered in the iscsi-target metrics layer;
// this struct provides the raw observations for GaugeFunc export.
type atomicHistogram struct {
count atomic.Uint64
sum atomic.Int64
}
func (h *atomicHistogram) record(val int64) {
h.count.Add(1)
h.sum.Add(val)
}
func (h *atomicHistogram) snapshot() (uint64, int64) {
return h.count.Load(), h.sum.Load()
}