admin: accept a list of collections in the task collection filter (#10953)

The collection filter was parsed twice with two syntaxes: the master-side
volume listing compiled the whole string as one regex, while EC encode and
EC balance detection split it on commas and matched each entry as a
wildcard. A volume had to pass both, so "collection-a,collection-b" matched
nothing (no collection is named that), and the ALL_COLLECTIONS sentinel,
which the master side skips, dropped every volume at the task side.

Parse it once, in one place: a comma-separated list where an entry is a
name with optional * and ? wildcards, or a regex when it carries regex
syntax. A regex entry now has to match the whole name unless it anchors
itself, so listing a collection no longer picks up its longer namesakes.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-08-25 15:13:06 -07:00
1 parent ef4c9d9178
commit e482e67971
17 files changed
+235 -48

No files matched your search

+3 -3
View File
@@ -123,9 +123,9 @@ func GetConfigSpec() base.ConfigSpec {
DefaultValue: "",
Required: false,
DisplayName: "Collection Filter",
Description: "Only process volumes from specific collections",
HelpText: "Leave empty to process all collections, or specify collection name",
Placeholder: "my_collection",
Description: "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
HelpText: "Leave empty to process all collections, or list the collections to process",
Placeholder: "pictures,videos",
InputType: "text",
CSSClasses: "form-control",
},
@@ -14,10 +14,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -69,7 +69,10 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
glog.Warningf("EC Detection: replica placement data-center digit (%d) is ignored for EC; only rack/node digits are honored", replicaPlacement.DiffDataCenterCount)
}
allowedCollections := wildcard.CompileWildcardMatchers(ecConfig.CollectionFilter)
allowedCollections, err := pluginworker.CompileCollectionMatcher(ecConfig.CollectionFilter)
if err != nil {
return nil, false, err
}
// Cluster node count for the min-node safety gate (mirrors the shell ec.encode
// guard that refuses to encode when nodes < parity shards, so shards cannot be
@@ -187,7 +190,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
}
// Check collection filter if specified
if len(allowedCollections) > 0 && !wildcard.MatchesAnyWildcard(allowedCollections, metric.Collection) {
if !allowedCollections.Matches(metric.Collection) {
skippedCollectionFilter++
continue
}
@@ -18,7 +18,7 @@ import (
func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
// 14 nodes × 2 disks per node (hdd + ssd) — enough SSD slots alone
// for a 10+4 layout with one-shard-per-(server,disk) diversity.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0)
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 1,
@@ -53,7 +53,7 @@ func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
func TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce(t *testing.T) {
// Start with every node carrying both HDD and SSD, then strip SSD
// from all but the first node so the SSD pool is too small alone.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0)
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
topo := activeTopology.GetTopologyInfo()
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
@@ -16,7 +16,7 @@ import (
)
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0)
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 1,
@@ -207,7 +207,7 @@ func TestCountExistingEcShardsForVolume(t *testing.T) {
}
func TestDetectionContextCancellation(t *testing.T) {
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0)
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0, "")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(50)
@@ -218,9 +218,44 @@ func TestDetectionContextCancellation(t *testing.T) {
require.ErrorIs(t, err, context.Canceled)
}
// The admin UI collection filter takes a list, and the two mode sentinels must
// not be read as a collection name.
func TestDetectionCollectionFilter(t *testing.T) {
for _, tc := range []struct {
filter string
want int
}{
{filter: "", want: 3},
{filter: "ALL_COLLECTIONS", want: 3},
{filter: "collection-a", want: 3},
{filter: "collection-a,collection-b", want: 3},
{filter: " collection-b , collection-a ", want: 3},
{filter: "collection-*", want: 3},
{filter: "collection-b,collection-c", want: 0},
{filter: "collection-ab", want: 0},
} {
t.Run(tc.filter, func(t *testing.T) {
// A fresh topology per case: planned destinations reserve capacity.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "collection-a")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(3)
for _, metric := range metrics {
metric.Collection = "collection-a"
}
config := NewDefaultConfig()
config.CollectionFilter = tc.filter
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
require.NoError(t, err)
assert.Len(t, results, tc.want)
})
}
}
func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
// One node per shard so each shard gets its own disk (#9369).
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0)
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(3)
@@ -285,7 +320,7 @@ func TestPlanECDestinationsSpreadsAcrossPhysicalDisks(t *testing.T) {
}
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1)
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 2,
@@ -411,7 +446,7 @@ func buildVolumeMetricsForIDs(count int) []*types.VolumeHealthMetrics {
return metrics
}
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64) *topology.ActiveTopology {
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64, collection string) *topology.ActiveTopology {
t.Helper()
activeTopology := topology.NewActiveTopology(10)
@@ -427,7 +462,7 @@ func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVol
for vid := 1; vid <= 200; vid++ {
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
Id: uint32(vid),
Collection: "",
Collection: collection,
DiskId: uint32(diskIndex),
})
}
@@ -14,7 +14,6 @@ import (
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
@@ -78,7 +77,7 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
{
Name: "collection_filter",
Label: "Collection Filter",
Description: "Only detect erasure coding opportunities in this collection when set.",
Description: "Only erasure code volumes in matching collections. Comma-separated list of names, wildcards, or regex patterns.",
Placeholder: "all collections",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
@@ -296,7 +295,10 @@ func emitErasureCodingDetectionDecisionTrace(
quietThreshold := time.Duration(taskConfig.QuietForSeconds) * time.Second
minSizeBytes := uint64(taskConfig.MinSizeMB) * 1024 * 1024
allowedCollections := wildcard.CompileWildcardMatchers(taskConfig.CollectionFilter)
allowedCollections, err := pluginworker.CompileCollectionMatcher(taskConfig.CollectionFilter)
if err != nil {
return err
}
volumeGroups := make(map[uint32][]*workertypes.VolumeHealthMetrics)
for _, metric := range metrics {
@@ -334,7 +336,7 @@ func emitErasureCodingDetectionDecisionTrace(
skippedTooSmall++
continue
}
if len(allowedCollections) > 0 && !wildcard.MatchesAnyWildcard(allowedCollections, metric.Collection) {
if !allowedCollections.Matches(metric.Collection) {
skippedCollectionFilter++
continue
}