mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-10 22:08:48 +01:00
fix(ec): bring ec.encode worker and EC/volume helpers to parity with shell (#9599)
* refactor(volume): extract replica sync/select into shared volume_replica package Move the volume replica reconciliation helpers (status, union builder, SyncAndSelectBestReplica, ReadNeedleMeta) out of the shell into a new weed/storage/volume_replica package so both the shell (ec.encode, volume.tier.move, volume.check.disk) and the EC encode worker can reuse them. No behavior change. * fix(ec): bring ec.encode worker to parity with the shell - Sync replicas and encode the most-complete one (via the shared volume_replica.SyncAndSelectBestReplica) instead of a possibly-stale replica, marking all replicas readonly first. Prevents silent data loss when a stale replica is encoded and the originals deleted. - Skip remote/tiered volumes in detection (shell ec.encode excludes them). - Min-node safety gate: refuse to encode when cluster nodes < parity shards. - Align default thresholds with the shell (fullness 0.95, quiet 1h). * fix(vacuum): plugin path honors min_volume_age_seconds override deriveVacuumConfig hard-coded MinVolumeAgeSeconds=0, dropping any configured value. Read it from worker config (default 0, matching the shell/master vacuum which has no age gate) so an explicit override is honored. * address review feedback - config.go: align GetConfigSpec schema defaults (quiet_for_seconds=3600, fullness_ratio=0.95) with the runtime defaults so UI/bootstrap flows match the shell (coderabbitai). - ec_task.go: roll back readonly when markReplicasReadonly fails partway, so already-marked replicas don't stay readonly (coderabbitai). - volume_replica: pass the caller's replica statuses into buildUnionReplica instead of re-fetching them, and skip the per-needle ReadNeedleMeta RPC when the source replica is read-only (gemini-code-assist). * test(plugin_workers/ec): make fixtures eligible under the new defaults The default EC encode thresholds were raised to match the shell (fullness 0.95, quiet 1h), but the plugin-worker integration fixtures still used 90%-full / 10-minute-old volumes, so detection found no eligible volumes and the tests failed in CI. Bump the eligible fixtures to 96% full and 2h old.
This commit is contained in:
1 parent
3f6410fdc3
commit
cd15ae1395
15 files changed
+238
-134
No files matched your search
@@ -52,12 +52,19 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
|
||||
skippedCollectionFilter := 0
|
||||
skippedQuietTime := 0
|
||||
skippedFullness := 0
|
||||
skippedRemote := 0
|
||||
skippedTooFewNodes := 0
|
||||
consecutivePlanningFailures := 0
|
||||
|
||||
var planner *ecPlacementPlanner
|
||||
|
||||
allowedCollections := wildcard.CompileWildcardMatchers(ecConfig.CollectionFilter)
|
||||
|
||||
// Cluster node count for the min-node safety gate (mirrors the shell ec.encode
|
||||
// guard that refuses to encode when nodes < parity shards, so shards cannot be
|
||||
// spread for fault tolerance).
|
||||
clusterNodeCount := countTopologyNodes(clusterInfo.ActiveTopology)
|
||||
|
||||
// Group metrics by VolumeID to handle replicas and select canonical server
|
||||
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
@@ -163,6 +170,21 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip remote/tiered volumes: encoding them would lose the tiering. The
|
||||
// shell ec.encode excludes remote volumes for the same reason.
|
||||
if metric.HasRemoteCopy {
|
||||
skippedRemote++
|
||||
continue
|
||||
}
|
||||
|
||||
// Min-node safety gate: don't encode when the cluster has fewer nodes than
|
||||
// this collection's parity shards — shards could not be spread to tolerate
|
||||
// failures. Mirrors the shell ec.encode guard (node count < parity shards).
|
||||
if clusterNodeCount > 0 && clusterNodeCount < erasure_coding.ParityShardsCount {
|
||||
skippedTooFewNodes++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check quiet duration and fullness criteria
|
||||
if metric.Age >= quietThreshold && metric.FullnessRatio >= ecConfig.FullnessRatio {
|
||||
if ctx != nil {
|
||||
@@ -373,8 +395,8 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
|
||||
// Log debug summary if no tasks were created
|
||||
if len(results) == 0 && len(metrics) > 0 && !stoppedEarly {
|
||||
totalVolumes := len(metrics)
|
||||
glog.V(1).Infof("EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
|
||||
totalVolumes, skippedAlreadyEC, skippedTooSmall, skippedCollectionFilter, skippedQuietTime, skippedFullness)
|
||||
glog.V(1).Infof("EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full, %d remote, %d too few nodes)",
|
||||
totalVolumes, skippedAlreadyEC, skippedTooSmall, skippedCollectionFilter, skippedQuietTime, skippedFullness, skippedRemote, skippedTooFewNodes)
|
||||
|
||||
// Show details for first few volumes
|
||||
for i, metric := range metrics {
|
||||
@@ -658,6 +680,25 @@ func (p *ecPlacementPlanner) buildCandidateSets(shardsNeeded int) [][]*placement
|
||||
|
||||
// planECDestinations plans the destinations for erasure coding operation.
|
||||
// dataShards/parityShards are parameters so callers can drive non-10+4 ratios.
|
||||
// countTopologyNodes counts volume-server nodes in the active topology, used by
|
||||
// the min-node safety gate.
|
||||
func countTopologyNodes(at *topology.ActiveTopology) int {
|
||||
if at == nil {
|
||||
return 0
|
||||
}
|
||||
topo := at.GetTopologyInfo()
|
||||
if topo == nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, dc := range topo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
n += len(rack.DataNodeInfos)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthMetrics, ecConfig *Config, dataShards, parityShards int) (*topology.MultiDestinationPlan, error) {
|
||||
if planner == nil || planner.activeTopology == nil {
|
||||
return nil, fmt.Errorf("active topology not available for EC placement")
|
||||
|
||||
Reference in new issue
Block a user