mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-18 12:08:32 +02:00
* shell: make every -collection flag parse the same way The filter vocabulary (comma-separated names, wildcards, regexes, "_default" for the empty-named collection) already existed in wildcard.CompileCollectionMatcher, but several commands still did their own exact-match comparison or sent the raw flag value to servers. fs.mergeVolumes used a literal string compare, so -collection=_default silently produced an empty merge plan; it now uses the shared matcher while keeping "*" as the match-all default. volume.fsck and volume.mark filter through the matcher too, and volume.vacuum resolves the filter client-side into per-collection and per-volume requests, since no Collection value scopes the VacuumVolume RPC to the unnamed collection alone. Commands that put one concrete collection name on the wire normalize "_default" to "" (volume.grow, fs.configure, mq.topic.compact, collection.delete, ec.rebuild, ec.balance), and the tier -volumeId commands resolve the volume's actual collection from topology first so patterns and named-collection requests pass the volume server's collection check. * shell: spell the default-collection keyword via CollectionDefault * shell: address review findings on collection filter handling volume.fsck keeps a set of every volume the master knows before any filtering, so a filer entry whose volume was excluded by -collection or -volumeId is skipped instead of reported "volume not found" and deleted by -reallyDeleteFilerEntries. The collection filter also no longer scopes the filer walk to /buckets/<collection>: fs.configure can bind a collection to any path prefix, and a narrower walk could mark live chunks as orphans under -reallyDeleteFromVolume. ec.rebuild and ec.balance resolve -collection through the new wildcard.ExpandCollectionFilter, which passes literal names through even when the master does not list them — ec.rebuild must be able to recover a collection whose shards all lost their index and never registered. fs.configure and mq.topic.compact now reject _default outright: an empty Collection means "unset" in both the path-rule merge and the filer's storage-option fallback chain, so the keyword would have been silently ignored. The unnamed-collection vacuum sweep keeps the master's low-disk-space exception: a volume whose every replica is read-only is probed with VacuumVolumeCheck and stays eligible when a replica reports DiskSpaceLow, instead of being dropped outright.
168 lines
4.9 KiB
Go
168 lines
4.9 KiB
Go
package shell
|
|
|
|
import (
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
func TestCollectVolumeMarkTargetsByCollection(t *testing.T) {
|
|
topo := &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{
|
|
Id: "node-a:8080",
|
|
Address: "10.0.0.1:8080",
|
|
GrpcPort: 19080,
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Collection: "photos"},
|
|
{Id: 2, Collection: "docs"},
|
|
},
|
|
},
|
|
"ssd": {
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 3, Collection: "photos"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Id: "node-b:8080",
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Collection: "photos"},
|
|
},
|
|
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
|
{Id: 4, Collection: "photos"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
targets, err := collectVolumeMarkTargetsByCollection(topo, "photos")
|
|
if err != nil {
|
|
t.Fatalf("collectVolumeMarkTargetsByCollection: %v", err)
|
|
}
|
|
|
|
got := map[string]int{}
|
|
for _, target := range targets {
|
|
got[string(target.sourceVolumeServer)+"#"+target.volumeId.String()]++
|
|
}
|
|
want := map[string]int{
|
|
string(pb.ServerAddress("10.0.0.1:8080.19080")) + "#" + needle.VolumeId(1).String(): 1,
|
|
string(pb.ServerAddress("10.0.0.1:8080.19080")) + "#" + needle.VolumeId(3).String(): 1,
|
|
string(pb.ServerAddress("node-b:8080")) + "#" + needle.VolumeId(1).String(): 1,
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("got %v targets, want %v", got, want)
|
|
}
|
|
for key, wantCount := range want {
|
|
if got[key] != wantCount {
|
|
t.Fatalf("target %s count %d, want %d; all targets: %v", key, got[key], wantCount, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCollectVolumeMarkTargetsByCollectionMatchesDefaultCollection(t *testing.T) {
|
|
topo := &master_pb.TopologyInfo{
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{
|
|
Id: "node-a:8080",
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Collection: ""},
|
|
{Id: 2, Collection: "photos"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
targets, err := collectVolumeMarkTargetsByCollection(topo, CollectionDefault)
|
|
if err != nil {
|
|
t.Fatalf("collectVolumeMarkTargetsByCollection: %v", err)
|
|
}
|
|
if len(targets) != 1 || targets[0].volumeId != needle.VolumeId(1) {
|
|
t.Fatalf("expected only volume 1 from the default collection, got %v", targets)
|
|
}
|
|
}
|
|
|
|
func TestCollectVolumeMarkTargetsByCollectionRejectsEmptyCollection(t *testing.T) {
|
|
_, err := collectVolumeMarkTargetsByCollection(&master_pb.TopologyInfo{}, "")
|
|
if err == nil || !strings.Contains(err.Error(), "collection is required") {
|
|
t.Fatalf("expected collection required error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCollectVolumeMarkTargetsByCollectionReportsMissingCollection(t *testing.T) {
|
|
_, err := collectVolumeMarkTargetsByCollection(&master_pb.TopologyInfo{}, "missing")
|
|
if err == nil || !strings.Contains(err.Error(), "no volumes match collection missing") {
|
|
t.Fatalf("expected missing collection error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestVolumeMarkRejectsCollectionWithNodeOrVolumeId(t *testing.T) {
|
|
err := (&commandVolumeMark{}).Do([]string{
|
|
"-collection", "photos",
|
|
"-node", "127.0.0.1:8080",
|
|
"-readonly",
|
|
}, nil, io.Discard)
|
|
if err == nil || !strings.Contains(err.Error(), "cannot use -collection with -node or -volumeId") {
|
|
t.Fatalf("expected collection conflict error, got %v", err)
|
|
}
|
|
|
|
err = (&commandVolumeMark{}).Do([]string{
|
|
"-collection", "photos",
|
|
"-volumeId", "1",
|
|
"-readonly",
|
|
}, nil, io.Discard)
|
|
if err == nil || !strings.Contains(err.Error(), "cannot use -collection with -node or -volumeId") {
|
|
t.Fatalf("expected collection conflict error, got %v", err)
|
|
}
|
|
|
|
err = (&commandVolumeMark{}).Do([]string{
|
|
"-collection", "photos",
|
|
"-volumeId", "0",
|
|
"-readonly",
|
|
}, nil, io.Discard)
|
|
if err == nil || !strings.Contains(err.Error(), "cannot use -collection with -node or -volumeId") {
|
|
t.Fatalf("expected collection conflict error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestVolumeMarkRejectsEmptyCollectionFlag(t *testing.T) {
|
|
err := (&commandVolumeMark{}).Do([]string{
|
|
"-collection", "",
|
|
"-readonly",
|
|
}, nil, io.Discard)
|
|
if err == nil || !strings.Contains(err.Error(), "collection is required") {
|
|
t.Fatalf("expected collection required error, got %v", err)
|
|
}
|
|
}
|