mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-18 20:18:27 +02:00
Chris Lu
b0ab4c4e1e
* shell: make every -collection flag parse the same way The filter vocabulary (comma-separated names, wildcards, regexes, "_default" for the empty-named collection) already existed in wildcard.CompileCollectionMatcher, but several commands still did their own exact-match comparison or sent the raw flag value to servers. fs.mergeVolumes used a literal string compare, so -collection=_default silently produced an empty merge plan; it now uses the shared matcher while keeping "*" as the match-all default. volume.fsck and volume.mark filter through the matcher too, and volume.vacuum resolves the filter client-side into per-collection and per-volume requests, since no Collection value scopes the VacuumVolume RPC to the unnamed collection alone. Commands that put one concrete collection name on the wire normalize "_default" to "" (volume.grow, fs.configure, mq.topic.compact, collection.delete, ec.rebuild, ec.balance), and the tier -volumeId commands resolve the volume's actual collection from topology first so patterns and named-collection requests pass the volume server's collection check. * shell: spell the default-collection keyword via CollectionDefault * shell: address review findings on collection filter handling volume.fsck keeps a set of every volume the master knows before any filtering, so a filer entry whose volume was excluded by -collection or -volumeId is skipped instead of reported "volume not found" and deleted by -reallyDeleteFilerEntries. The collection filter also no longer scopes the filer walk to /buckets/<collection>: fs.configure can bind a collection to any path prefix, and a narrower walk could mark live chunks as orphans under -reallyDeleteFromVolume. ec.rebuild and ec.balance resolve -collection through the new wildcard.ExpandCollectionFilter, which passes literal names through even when the master does not list them — ec.rebuild must be able to recover a collection whose shards all lost their index and never registered. fs.configure and mq.topic.compact now reject _default outright: an empty Collection means "unset" in both the path-rule merge and the filer's storage-option fallback chain, so the keyword would have been silently ignored. The unnamed-collection vacuum sweep keeps the master's low-disk-space exception: a volume whose every replica is read-only is probed with VacuumVolumeCheck and stays eligible when a replica reports DiskSpaceLow, instead of being dropped outright.
104 lines
3.2 KiB
Go
104 lines
3.2 KiB
Go
package shell
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer_client"
|
|
"github.com/seaweedfs/seaweedfs/weed/mq/logstore"
|
|
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
|
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
|
|
)
|
|
|
|
func init() {
|
|
Commands = append(Commands, &commandMqTopicCompact{})
|
|
}
|
|
|
|
type commandMqTopicCompact struct {
|
|
}
|
|
|
|
func (c *commandMqTopicCompact) Name() string {
|
|
return "mq.topic.compact"
|
|
}
|
|
|
|
func (c *commandMqTopicCompact) Help() string {
|
|
return `compact the topic storage into parquet format
|
|
|
|
Example:
|
|
mq.topic.compact -namespace <namespace> -topic <topic_name> -timeAgo <time_ago>
|
|
|
|
`
|
|
}
|
|
|
|
func (c *commandMqTopicCompact) HasTag(tag CommandTag) bool {
|
|
return ResourceHeavy == tag
|
|
}
|
|
|
|
func (c *commandMqTopicCompact) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
|
|
|
// parse parameters
|
|
mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
|
namespace := mqCommand.String("namespace", "", "namespace name")
|
|
topicName := mqCommand.String("topic", "", "topic name")
|
|
timeAgo := mqCommand.Duration("timeAgo", 2*time.Minute, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
|
|
replication := mqCommand.String("replication", "", "replication type")
|
|
collection := mqCommand.String("collection", "", "optional collection name")
|
|
dataCenter := mqCommand.String("dataCenter", "", "optional data center name")
|
|
diskType := mqCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
|
maxMB := mqCommand.Int("maxMB", 4, "split files larger than the limit")
|
|
|
|
if err := mqCommand.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
// The AssignVolumeRequest collection falls back to the path rule and the
|
|
// filer default when empty, so the empty-named collection cannot be
|
|
// requested explicitly.
|
|
if *collection == CollectionDefault {
|
|
return fmt.Errorf("%s is not a usable collection name here; omit -collection to inherit the configured collection", CollectionDefault)
|
|
}
|
|
|
|
storagePreference := &operation.StoragePreference{
|
|
Replication: *replication,
|
|
Collection: *collection,
|
|
DataCenter: *dataCenter,
|
|
DiskType: *diskType,
|
|
MaxMB: *maxMB,
|
|
}
|
|
|
|
// read topic configuration
|
|
fca := filer_client.NewFilerClientAccessor(
|
|
[]pb.ServerAddress{commandEnv.option.FilerAddress},
|
|
commandEnv.option.GrpcDialOption,
|
|
)
|
|
t := topic.NewTopic(*namespace, *topicName)
|
|
topicConf, err := fca.ReadTopicConfFromFiler(t)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// get record type - prefer flat schema if available
|
|
var recordType *schema_pb.RecordType
|
|
if topicConf.GetMessageRecordType() != nil {
|
|
// New flat schema format - use directly
|
|
recordType = topicConf.GetMessageRecordType()
|
|
}
|
|
recordType = schema.NewRecordTypeBuilder(recordType).
|
|
WithField(logstore.SW_COLUMN_NAME_TS, schema.TypeInt64).
|
|
WithField(logstore.SW_COLUMN_NAME_KEY, schema.TypeBytes).
|
|
RecordTypeEnd()
|
|
|
|
// compact the topic partition versions
|
|
if err = logstore.CompactTopicPartitions(commandEnv, t, *timeAgo, recordType, storagePreference); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|