package weed_server import ( "context" "errors" "fmt" "time" "github.com/prometheus/client_golang/prometheus" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.ScrubVolumeRequest) (*volume_server_pb.ScrubVolumeResponse, error) { if err := vs.checkGrpcAdminAuth(ctx); err != nil { return nil, err } vids := []needle.VolumeId{} if len(req.GetVolumeIds()) == 0 { for _, l := range vs.store.Locations { vids = append(vids, l.VolumeIds()...) } } else { for _, vid := range req.GetVolumeIds() { vids = append(vids, needle.VolumeId(vid)) } } var details []string var totalVolumes, totalFiles uint64 var brokenVolumes []*storage.Volume var brokenVolumeIds []uint32 for _, vid := range vids { v := vs.store.GetVolume(vid) if v == nil { return nil, fmt.Errorf("volume id %d not found", vid) } var files int64 var serrs []error switch m := req.GetMode(); m { case volume_server_pb.VolumeScrubMode_INDEX: files, serrs = v.ScrubIndex() case volume_server_pb.VolumeScrubMode_LOCAL, volume_server_pb.VolumeScrubMode_READS: // both are equivalent to FULL for regular volumes: there are no shards to // stay local to, and nothing to reconstruct from fallthrough case volume_server_pb.VolumeScrubMode_FULL: files, serrs = v.Scrub() default: return nil, fmt.Errorf("unsupported volume scrub mode %d", m) } totalVolumes += 1 totalFiles += uint64(files) if len(serrs) != 0 { brokenVolumes = append(brokenVolumes, v) brokenVolumeIds = append(brokenVolumeIds, uint32(v.Id)) for _, err := range serrs { details = append(details, err.Error()) } } } errs := []error{} if req.GetMarkBrokenVolumesReadonly() { for _, v := range brokenVolumes { if err := vs.makeVolumeReadonly(ctx, v, false, true); err != nil { errs = append(errs, err) details = append(details, err.Error()) } else { details = append(details, fmt.Sprintf("volume %d is now read-only", v.Id)) } } } scrubLabels := prometheus.Labels{"mode": req.GetMode().String()} stats.VolumeServerScrubLastTimeSeconds.With(scrubLabels).Set(float64(time.Now().Unix())) stats.VolumeServerScrubVolumeFailures.With(scrubLabels).Add(float64(len(brokenVolumes))) if len(errs) != 0 { return nil, errors.Join(errs...) } res := &volume_server_pb.ScrubVolumeResponse{ TotalVolumes: totalVolumes, TotalFiles: totalFiles, BrokenVolumeIds: brokenVolumeIds, Details: details, } return res, nil } func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb.ScrubEcVolumeRequest) (*volume_server_pb.ScrubEcVolumeResponse, error) { if err := vs.checkGrpcAdminAuth(ctx); err != nil { return nil, err } if m := req.GetMode(); req.GetForceDeletedNeedlesCheck() && m != volume_server_pb.VolumeScrubMode_FULL && m != volume_server_pb.VolumeScrubMode_READS { return nil, fmt.Errorf("deleted needle checks are only supported for FULL and READS scrubs") } vids := []needle.VolumeId{} if len(req.GetVolumeIds()) == 0 { for _, l := range vs.store.Locations { vids = append(vids, l.EcVolumeIds()...) } } else { for _, vid := range req.GetVolumeIds() { vids = append(vids, needle.VolumeId(vid)) } } var details []string var totalVolumes, totalFiles uint64 var brokenVolumeIds []uint32 var brokenShardInfos []*volume_server_pb.EcShardInfo for _, vid := range vids { v, found := vs.store.FindEcVolume(vid) if !found { return nil, fmt.Errorf("EC volume id %d not found", vid) } var files int64 var shardInfos []*volume_server_pb.EcShardInfo var serrs []error switch m := req.GetMode(); m { case volume_server_pb.VolumeScrubMode_INDEX: // index scrubs do not verify individual EC shards files, serrs = v.ScrubIndex() case volume_server_pb.VolumeScrubMode_LOCAL: files, shardInfos, serrs = v.ScrubLocal() case volume_server_pb.VolumeScrubMode_FULL, volume_server_pb.VolumeScrubMode_READS: files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId, m, req.GetForceDeletedNeedlesCheck()) case volume_server_pb.VolumeScrubMode_CHECKSUM: // Verify each local shard's raw bytes against the bitrot sidecar, // exercising cold parity shards. Read-only. ChecksumScrub's first // return is blocks scanned, not files — discard it so TotalFiles // (a needle/file count) isn't inflated by the block count. _, shardInfos, serrs = v.ChecksumScrub() default: return nil, fmt.Errorf("unsupported EC volume scrub mode %d", m) } totalVolumes += 1 totalFiles += uint64(files) if len(serrs) != 0 || len(shardInfos) != 0 { brokenVolumeIds = append(brokenVolumeIds, uint32(v.VolumeId)) brokenShardInfos = append(brokenShardInfos, shardInfos...) for _, err := range serrs { details = append(details, err.Error()) } } } scrubLabels := prometheus.Labels{"mode": req.GetMode().String()} stats.VolumeServerScrubLastTimeSeconds.With(scrubLabels).Set(float64(time.Now().Unix())) stats.VolumeServerScrubVolumeFailures.With(scrubLabels).Add(float64(len(brokenVolumeIds))) stats.VolumeServerScrubShardFailures.With(scrubLabels).Add(float64(len(brokenShardInfos))) res := &volume_server_pb.ScrubEcVolumeResponse{ TotalVolumes: totalVolumes, TotalFiles: totalFiles, BrokenVolumeIds: brokenVolumeIds, BrokenShardInfos: brokenShardInfos, Details: details, } return res, nil }