mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-06 03:57:48 +01:00
Durability mode implementation (sync_all, sync_quorum, best_effort): - DurabilityMode type with superblock persistence, parse/validate/string - MakeDistributedSync mode-aware barrier enforcement in dist_group_commit - blockerr sentinel package (ErrDurabilityBarrierFailed, ErrDurabilityQuorumLost) - gRPC create path: mode validation, idempotent create consistency, partial cleanup - F1: strict mode rejects partial replica provisioning with cleanup - F3: empty heartbeat does not overwrite persisted strict mode - F4: SCSI error mapping uses errors.Is sentinels (not string matching) - Proto/wire/blockapi/CLI/UI plumbing for durability_mode field - Observability dashboard: cluster health cards + per-volume columns Testrunner platform (YAML-driven integration test framework): - Engine, parser, registry, reporter (JUnit XML + HTML), metrics scraping - 52 registered actions: block, iSCSI, I/O, fault injection, assertions - Baseline regression framework with 7 hard-fail conditions - 15 YAML scenarios (smoke, crash, HA, fault, consistency, snapshot) - 49 unit tests for testrunner internals QA adversarial suite (21 tests, all PASS): - Idempotent create mode/RF mismatch detection - Heartbeat mode downgrade prevention (F3) - sync_all/sync_quorum partial replica enforcement (F1) - Concurrent create race safety - Failover/expand mode preservation - Cleanup resilience when delete fails - Master restart auto-register mode handling - Superblock roundtrip all 3 modes - Validate edge cases (mode×RF matrix) - RequiredReplicas quorum math verification - Sentinel error categorization Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
124 lines
3.8 KiB
Go
124 lines
3.8 KiB
Go
package blockvol
|
|
|
|
import (
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
// EngineMetrics collects subsystem-level counters for Prometheus export.
|
|
// All fields are safe for concurrent access.
|
|
type EngineMetrics struct {
|
|
// Flusher
|
|
FlusherBytesTotal atomic.Uint64
|
|
FlusherFlushesTotal atomic.Uint64
|
|
FlusherErrorsTotal atomic.Uint64
|
|
flusherLatencyNs atomicHistogram // flush cycle duration
|
|
|
|
// Group Commit
|
|
GroupCommitBatchTotal atomic.Uint64 // alias for GroupCommitter.syncCount
|
|
groupCommitBatchSize atomicHistogram
|
|
groupCommitWaitNs atomicHistogram
|
|
|
|
// WAL Shipper
|
|
WALShippedEntriesTotal atomic.Uint64
|
|
WALBarrierRequestsTotal atomic.Uint64
|
|
WALFailedBarriersTotal atomic.Uint64
|
|
walBarrierLatencyNs atomicHistogram
|
|
|
|
// Scrub
|
|
ScrubPassesTotal atomic.Uint64
|
|
ScrubErrorsTotal atomic.Uint64
|
|
scrubDurationNs atomicHistogram
|
|
|
|
// Durability (CP8-3-1)
|
|
DurabilityBarrierFailedTotal atomic.Uint64 // sync_all barrier failures
|
|
DurabilityQuorumLostTotal atomic.Uint64 // sync_quorum quorum lost
|
|
}
|
|
|
|
// NewEngineMetrics creates an EngineMetrics instance.
|
|
func NewEngineMetrics() *EngineMetrics {
|
|
return &EngineMetrics{}
|
|
}
|
|
|
|
// RecordFlusherFlush records a completed flush cycle.
|
|
func (m *EngineMetrics) RecordFlusherFlush(bytesWritten uint64, dur time.Duration) {
|
|
m.FlusherBytesTotal.Add(bytesWritten)
|
|
m.FlusherFlushesTotal.Add(1)
|
|
m.flusherLatencyNs.record(dur.Nanoseconds())
|
|
}
|
|
|
|
// RecordFlusherError records a flush error.
|
|
func (m *EngineMetrics) RecordFlusherError() {
|
|
m.FlusherErrorsTotal.Add(1)
|
|
}
|
|
|
|
// RecordGroupCommitBatch records a group commit batch.
|
|
func (m *EngineMetrics) RecordGroupCommitBatch(batchSize int, waitDur time.Duration) {
|
|
m.GroupCommitBatchTotal.Add(1)
|
|
m.groupCommitBatchSize.record(int64(batchSize))
|
|
m.groupCommitWaitNs.record(waitDur.Nanoseconds())
|
|
}
|
|
|
|
// RecordWALShipped records a shipped WAL entry.
|
|
func (m *EngineMetrics) RecordWALShipped() {
|
|
m.WALShippedEntriesTotal.Add(1)
|
|
}
|
|
|
|
// RecordWALBarrier records a barrier request with its result.
|
|
func (m *EngineMetrics) RecordWALBarrier(dur time.Duration, failed bool) {
|
|
m.WALBarrierRequestsTotal.Add(1)
|
|
m.walBarrierLatencyNs.record(dur.Nanoseconds())
|
|
if failed {
|
|
m.WALFailedBarriersTotal.Add(1)
|
|
}
|
|
}
|
|
|
|
// RecordScrubPass records a completed scrub pass.
|
|
func (m *EngineMetrics) RecordScrubPass(dur time.Duration, errors int64) {
|
|
m.ScrubPassesTotal.Add(1)
|
|
m.ScrubErrorsTotal.Add(uint64(errors))
|
|
m.scrubDurationNs.record(dur.Nanoseconds())
|
|
}
|
|
|
|
// FlusherLatencySnapshot returns flusher latency stats (count, sum_ns).
|
|
func (m *EngineMetrics) FlusherLatencySnapshot() (count uint64, sumNs int64) {
|
|
return m.flusherLatencyNs.snapshot()
|
|
}
|
|
|
|
// GroupCommitBatchSizeSnapshot returns batch size stats.
|
|
func (m *EngineMetrics) GroupCommitBatchSizeSnapshot() (count uint64, sum int64) {
|
|
return m.groupCommitBatchSize.snapshot()
|
|
}
|
|
|
|
// GroupCommitWaitSnapshot returns group commit wait time stats.
|
|
func (m *EngineMetrics) GroupCommitWaitSnapshot() (count uint64, sumNs int64) {
|
|
return m.groupCommitWaitNs.snapshot()
|
|
}
|
|
|
|
// WALBarrierLatencySnapshot returns barrier latency stats.
|
|
func (m *EngineMetrics) WALBarrierLatencySnapshot() (count uint64, sumNs int64) {
|
|
return m.walBarrierLatencyNs.snapshot()
|
|
}
|
|
|
|
// ScrubDurationSnapshot returns scrub duration stats.
|
|
func (m *EngineMetrics) ScrubDurationSnapshot() (count uint64, sumNs int64) {
|
|
return m.scrubDurationNs.snapshot()
|
|
}
|
|
|
|
// atomicHistogram is a minimal lock-free histogram that tracks count and sum.
|
|
// Prometheus histograms are registered in the iscsi-target metrics layer;
|
|
// this struct provides the raw observations for GaugeFunc export.
|
|
type atomicHistogram struct {
|
|
count atomic.Uint64
|
|
sum atomic.Int64
|
|
}
|
|
|
|
func (h *atomicHistogram) record(val int64) {
|
|
h.count.Add(1)
|
|
h.sum.Add(val)
|
|
}
|
|
|
|
func (h *atomicHistogram) snapshot() (uint64, int64) {
|
|
return h.count.Load(), h.sum.Load()
|
|
}
|