mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-18 03:58:32 +02:00
Chris Lu
b0ab4c4e1e
* shell: make every -collection flag parse the same way The filter vocabulary (comma-separated names, wildcards, regexes, "_default" for the empty-named collection) already existed in wildcard.CompileCollectionMatcher, but several commands still did their own exact-match comparison or sent the raw flag value to servers. fs.mergeVolumes used a literal string compare, so -collection=_default silently produced an empty merge plan; it now uses the shared matcher while keeping "*" as the match-all default. volume.fsck and volume.mark filter through the matcher too, and volume.vacuum resolves the filter client-side into per-collection and per-volume requests, since no Collection value scopes the VacuumVolume RPC to the unnamed collection alone. Commands that put one concrete collection name on the wire normalize "_default" to "" (volume.grow, fs.configure, mq.topic.compact, collection.delete, ec.rebuild, ec.balance), and the tier -volumeId commands resolve the volume's actual collection from topology first so patterns and named-collection requests pass the volume server's collection check. * shell: spell the default-collection keyword via CollectionDefault * shell: address review findings on collection filter handling volume.fsck keeps a set of every volume the master knows before any filtering, so a filer entry whose volume was excluded by -collection or -volumeId is skipped instead of reported "volume not found" and deleted by -reallyDeleteFilerEntries. The collection filter also no longer scopes the filer walk to /buckets/<collection>: fs.configure can bind a collection to any path prefix, and a narrower walk could mark live chunks as orphans under -reallyDeleteFromVolume. ec.rebuild and ec.balance resolve -collection through the new wildcard.ExpandCollectionFilter, which passes literal names through even when the master does not list them — ec.rebuild must be able to recover a collection whose shards all lost their index and never registered. fs.configure and mq.topic.compact now reject _default outright: an empty Collection means "unset" in both the path-rule merge and the filer's storage-option fallback chain, so the keyword would have been silently ignored. The unnamed-collection vacuum sweep keeps the master's low-disk-space exception: a volume whose every replica is read-only is probed with VacuumVolumeCheck and stays eligible when a replica reports DiskSpaceLow, instead of being dropped outright.
90 lines
3.5 KiB
Go
90 lines
3.5 KiB
Go
package shell
|
|
|
|
import (
|
|
"flag"
|
|
"io"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
|
)
|
|
|
|
func init() {
|
|
Commands = append(Commands, &commandEcBalance{})
|
|
}
|
|
|
|
type commandEcBalance struct {
|
|
}
|
|
|
|
func (c *commandEcBalance) Name() string {
|
|
return "ec.balance"
|
|
}
|
|
|
|
func (c *commandEcBalance) Help() string {
|
|
return `balance all ec shards among all racks and volume servers
|
|
|
|
ec.balance [-c EACH_COLLECTION|<collection_name>] [-apply] [-dataCenter <data_center>] [-shardReplicaPlacement <replica_placement>] [-diskType <disk_type>] [-volumeIds <id>[,<id>...]]
|
|
|
|
Options:
|
|
-diskType: the disk type for EC shards (hdd, ssd, or empty for default hdd)
|
|
-volumeIds: only balance these ec volume ids, e.g. -volumeIds 123,456. The plan is
|
|
built as if no other ec volume existed, so nothing else is moved or deduplicated.
|
|
Ids without any ec shard in the selected collection, dataCenter and disk type are
|
|
an error.
|
|
|
|
Algorithm:
|
|
` + ecBalanceAlgorithmDescription
|
|
}
|
|
|
|
func (c *commandEcBalance) HasTag(CommandTag) bool {
|
|
return false
|
|
}
|
|
|
|
func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
|
balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
|
collection := balanceCommand.String("collection", string(wildcard.CollectionFilterEach), "comma-separated collection names, wildcards, or regex patterns, \""+CollectionDefault+"\" for the collection with no name, or \""+string(wildcard.CollectionFilterEach)+"\" for each collection")
|
|
dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
|
|
shardReplicaPlacement := balanceCommand.String("shardReplicaPlacement", "", "replica placement for EC shards, or master default if empty")
|
|
diskTypeStr := balanceCommand.String("diskType", "", "the disk type for EC shards (hdd, ssd, or empty for default hdd)")
|
|
volumeIdsStr := balanceCommand.String("volumeIds", "", "optional comma-separated list of ec volume ids to balance; defaults to all")
|
|
maxParallelization := balanceCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
|
|
ioBytePerSecond := balanceCommand.Int64("ioBytePerSecond", 0, "limit the speed of each shard copy; 0 falls back to the volume server's maintenance rate")
|
|
applyBalancing := balanceCommand.Bool("apply", false, "apply the balancing plan")
|
|
// TODO: remove this alias
|
|
applyBalancingAlias := balanceCommand.Bool("force", false, "apply the balancing plan (alias for -apply)")
|
|
|
|
if err = balanceCommand.Parse(args); err != nil {
|
|
return nil
|
|
}
|
|
|
|
handleDeprecatedForceFlag(writer, balanceCommand, applyBalancingAlias, applyBalancing)
|
|
infoAboutSimulationMode(writer, *applyBalancing, "-apply")
|
|
|
|
var volumeIds []needle.VolumeId
|
|
if *volumeIdsStr != "" {
|
|
if volumeIds, err = parseVolumeIdsFlag(*volumeIdsStr); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err = commandEnv.confirmIsLocked(args); err != nil {
|
|
return
|
|
}
|
|
|
|
collections, err := resolveEcCollections(commandEnv, *collection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
glog.V(1).Infof("balanceEcVolumes collections %+v\n", len(collections))
|
|
|
|
rp, err := parseReplicaPlacementArg(commandEnv, *shardReplicaPlacement)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
diskType := types.ToDiskType(*diskTypeStr)
|
|
|
|
return EcBalance(commandEnv, collections, *dc, rp, diskType, *maxParallelization, *ioBytePerSecond, *applyBalancing, nil, volumeIds)
|
|
}
|