mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-18 12:08:32 +02:00
shell: make every -collection flag parse the same way (#11794)
* shell: make every -collection flag parse the same way The filter vocabulary (comma-separated names, wildcards, regexes, "_default" for the empty-named collection) already existed in wildcard.CompileCollectionMatcher, but several commands still did their own exact-match comparison or sent the raw flag value to servers. fs.mergeVolumes used a literal string compare, so -collection=_default silently produced an empty merge plan; it now uses the shared matcher while keeping "*" as the match-all default. volume.fsck and volume.mark filter through the matcher too, and volume.vacuum resolves the filter client-side into per-collection and per-volume requests, since no Collection value scopes the VacuumVolume RPC to the unnamed collection alone. Commands that put one concrete collection name on the wire normalize "_default" to "" (volume.grow, fs.configure, mq.topic.compact, collection.delete, ec.rebuild, ec.balance), and the tier -volumeId commands resolve the volume's actual collection from topology first so patterns and named-collection requests pass the volume server's collection check. * shell: spell the default-collection keyword via CollectionDefault * shell: address review findings on collection filter handling volume.fsck keeps a set of every volume the master knows before any filtering, so a filer entry whose volume was excluded by -collection or -volumeId is skipped instead of reported "volume not found" and deleted by -reallyDeleteFilerEntries. The collection filter also no longer scopes the filer walk to /buckets/<collection>: fs.configure can bind a collection to any path prefix, and a narrower walk could mark live chunks as orphans under -reallyDeleteFromVolume. ec.rebuild and ec.balance resolve -collection through the new wildcard.ExpandCollectionFilter, which passes literal names through even when the master does not list them — ec.rebuild must be able to recover a collection whose shards all lost their index and never registered. fs.configure and mq.topic.compact now reject _default outright: an empty Collection means "unset" in both the path-rule merge and the filer's storage-option fallback chain, so the keyword would have been silently ignored. The unnamed-collection vacuum sweep keeps the master's low-disk-space exception: a volume whose every replica is read-only is probed with VacuumVolumeCheck and stays eligible when a replica reports DiskSpaceLow, instead of being dropped outright.
This commit is contained in:
1 parent
268424a5e1
commit
b0ab4c4e1e
23 files changed
+593
-71
No files matched your search
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -55,6 +56,8 @@ type commandVolumeFsck struct {
|
||||
writer io.Writer
|
||||
bucketsPath string
|
||||
collection *string
|
||||
collectionMatcher *wildcard.CollectionMatcher
|
||||
knownVolumeIds map[uint32]bool
|
||||
volumeIds map[uint32]bool
|
||||
scopedFilerPath string
|
||||
tempFolder string
|
||||
@@ -109,7 +112,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
|
||||
c.verbose = fsckCommand.Bool("v", false, "verbose mode")
|
||||
c.skipEcVolumes = fsckCommand.Bool("skipEcVolumes", false, "skip erasure coded volumes")
|
||||
c.findMissingChunksInFiler = fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"")
|
||||
c.collection = fsckCommand.String("collection", "", "the collection name")
|
||||
c.collection = fsckCommand.String("collection", "", "comma-separated collection names, wildcards, or regex patterns; empty or \"*\" matches all, \""+CollectionDefault+"\" the collection with no name")
|
||||
volumeIds := fsckCommand.String("volumeId", "", "comma separated the volume id")
|
||||
applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer. Currently this only works with default filerGroup.")
|
||||
c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging")
|
||||
@@ -147,6 +150,10 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
|
||||
c.env = commandEnv
|
||||
c.writer = writer
|
||||
|
||||
if c.collectionMatcher, err = wildcard.CompileCollectionMatcher(*c.collection); err != nil {
|
||||
return fmt.Errorf("invalid collection pattern %q: %w", *c.collection, err)
|
||||
}
|
||||
|
||||
c.bucketsPath, err = readFilerBucketsPath(commandEnv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read filer buckets path: %w", err)
|
||||
@@ -170,6 +177,16 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
|
||||
return fmt.Errorf("failed to collect all volume locations: %w", err)
|
||||
}
|
||||
|
||||
// Every volume the master knows about, before any filtering: a filer
|
||||
// entry pointing at a volume that was filtered out of this run is
|
||||
// excluded, not missing — it must never be reported or purged.
|
||||
c.knownVolumeIds = make(map[uint32]bool)
|
||||
for _, vidMap := range dataNodeVolumeIdToVInfo {
|
||||
for vid := range vidMap {
|
||||
c.knownVolumeIds[vid] = true
|
||||
}
|
||||
}
|
||||
|
||||
// If the operator specified -volumeId, reject unknown ids up front instead
|
||||
// of silently filtering them out and reporting "no orphan data". A silent
|
||||
// success on a missing volume looks identical to a clean volume and hides
|
||||
@@ -222,7 +239,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
|
||||
continue
|
||||
}
|
||||
}
|
||||
if *c.collection != "" && vinfo.collection != *c.collection {
|
||||
if !c.collectionMatcher.Matches(vinfo.collection) {
|
||||
delete(volumeIdToVInfo, volumeId)
|
||||
continue
|
||||
}
|
||||
@@ -375,6 +392,11 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A volume the master knows but this run filtered out is not a
|
||||
// missing volume: its files are healthy and stay untouched.
|
||||
if c.knownVolumeIds[i.vid] {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(c.writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
|
||||
if purgeAbsent {
|
||||
fmt.Fprintf(c.writer, "deleting path %s after volume not found\n", i.path)
|
||||
@@ -966,7 +988,11 @@ func (c *commandVolumeFsck) getCollectFilerFilePath() string {
|
||||
|
||||
func (c *commandVolumeFsck) resolveScopedFilerPath(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo) string {
|
||||
if *c.collection != "" {
|
||||
return fmt.Sprintf("%s/%s", c.bucketsPath, *c.collection)
|
||||
// fs.configure can bind a collection to any path prefix, so files
|
||||
// referencing the filtered volumes may live anywhere. A bucket-dir
|
||||
// walk could miss them and report their live chunks as orphans;
|
||||
// only a full walk is safe for a collection filter.
|
||||
return "/"
|
||||
}
|
||||
if len(c.volumeIds) == 0 {
|
||||
return "/"
|
||||
|
||||
Reference in new issue
Block a user