mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-25 22:40:44 +01:00
admin: accept a list of collections in the task collection filter (#10953)
The collection filter was parsed twice with two syntaxes: the master-side volume listing compiled the whole string as one regex, while EC encode and EC balance detection split it on commas and matched each entry as a wildcard. A volume had to pass both, so "collection-a,collection-b" matched nothing (no collection is named that), and the ALL_COLLECTIONS sentinel, which the master side skips, dropped every volume at the task side. Parse it once, in one place: a comma-separated list where an entry is a name with optional * and ? wildcards, or a regex when it carries regex syntax. A regex entry now has to match the whole name unless it anchors itself, so listing a collection no longer picks up its longer namesakes.
This commit is contained in:
1 parent
ef4c9d9178
commit
e482e67971
17 files changed
+235
-48
No files matched your search
@@ -123,9 +123,9 @@ func GetConfigSpec() base.ConfigSpec {
|
||||
DefaultValue: "",
|
||||
Required: false,
|
||||
DisplayName: "Collection Filter",
|
||||
Description: "Only process volumes from specific collections",
|
||||
HelpText: "Leave empty to process all collections, or specify collection name",
|
||||
Placeholder: "my_collection",
|
||||
Description: "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
|
||||
HelpText: "Leave empty to process all collections, or list the collections to process",
|
||||
Placeholder: "pictures,videos",
|
||||
InputType: "text",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
|
||||
@@ -14,10 +14,10 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
@@ -69,7 +69,10 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
|
||||
glog.Warningf("EC Detection: replica placement data-center digit (%d) is ignored for EC; only rack/node digits are honored", replicaPlacement.DiffDataCenterCount)
|
||||
}
|
||||
|
||||
allowedCollections := wildcard.CompileWildcardMatchers(ecConfig.CollectionFilter)
|
||||
allowedCollections, err := pluginworker.CompileCollectionMatcher(ecConfig.CollectionFilter)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// Cluster node count for the min-node safety gate (mirrors the shell ec.encode
|
||||
// guard that refuses to encode when nodes < parity shards, so shards cannot be
|
||||
@@ -187,7 +190,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
|
||||
}
|
||||
|
||||
// Check collection filter if specified
|
||||
if len(allowedCollections) > 0 && !wildcard.MatchesAnyWildcard(allowedCollections, metric.Collection) {
|
||||
if !allowedCollections.Matches(metric.Collection) {
|
||||
skippedCollectionFilter++
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
|
||||
// 14 nodes × 2 disks per node (hdd + ssd) — enough SSD slots alone
|
||||
// for a 10+4 layout with one-shard-per-(server,disk) diversity.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0)
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 1,
|
||||
@@ -53,7 +53,7 @@ func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
|
||||
func TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce(t *testing.T) {
|
||||
// Start with every node carrying both HDD and SSD, then strip SSD
|
||||
// from all but the first node so the SSD pool is too small alone.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0)
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
topo := activeTopology.GetTopologyInfo()
|
||||
for _, dc := range topo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0)
|
||||
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 1,
|
||||
@@ -207,7 +207,7 @@ func TestCountExistingEcShardsForVolume(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDetectionContextCancellation(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0)
|
||||
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0, "")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildVolumeMetricsForIDs(50)
|
||||
|
||||
@@ -218,9 +218,44 @@ func TestDetectionContextCancellation(t *testing.T) {
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
// The admin UI collection filter takes a list, and the two mode sentinels must
|
||||
// not be read as a collection name.
|
||||
func TestDetectionCollectionFilter(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
filter string
|
||||
want int
|
||||
}{
|
||||
{filter: "", want: 3},
|
||||
{filter: "ALL_COLLECTIONS", want: 3},
|
||||
{filter: "collection-a", want: 3},
|
||||
{filter: "collection-a,collection-b", want: 3},
|
||||
{filter: " collection-b , collection-a ", want: 3},
|
||||
{filter: "collection-*", want: 3},
|
||||
{filter: "collection-b,collection-c", want: 0},
|
||||
{filter: "collection-ab", want: 0},
|
||||
} {
|
||||
t.Run(tc.filter, func(t *testing.T) {
|
||||
// A fresh topology per case: planned destinations reserve capacity.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "collection-a")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
|
||||
metrics := buildVolumeMetricsForIDs(3)
|
||||
for _, metric := range metrics {
|
||||
metric.Collection = "collection-a"
|
||||
}
|
||||
|
||||
config := NewDefaultConfig()
|
||||
config.CollectionFilter = tc.filter
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
|
||||
// One node per shard so each shard gets its own disk (#9369).
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0)
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildVolumeMetricsForIDs(3)
|
||||
|
||||
@@ -285,7 +320,7 @@ func TestPlanECDestinationsSpreadsAcrossPhysicalDisks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1)
|
||||
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 2,
|
||||
@@ -411,7 +446,7 @@ func buildVolumeMetricsForIDs(count int) []*types.VolumeHealthMetrics {
|
||||
return metrics
|
||||
}
|
||||
|
||||
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64) *topology.ActiveTopology {
|
||||
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64, collection string) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
|
||||
@@ -427,7 +462,7 @@ func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVol
|
||||
for vid := 1; vid <= 200; vid++ {
|
||||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||||
Id: uint32(vid),
|
||||
Collection: "",
|
||||
Collection: collection,
|
||||
DiskId: uint32(diskIndex),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -78,7 +77,7 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
{
|
||||
Name: "collection_filter",
|
||||
Label: "Collection Filter",
|
||||
Description: "Only detect erasure coding opportunities in this collection when set.",
|
||||
Description: "Only erasure code volumes in matching collections. Comma-separated list of names, wildcards, or regex patterns.",
|
||||
Placeholder: "all collections",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
@@ -296,7 +295,10 @@ func emitErasureCodingDetectionDecisionTrace(
|
||||
|
||||
quietThreshold := time.Duration(taskConfig.QuietForSeconds) * time.Second
|
||||
minSizeBytes := uint64(taskConfig.MinSizeMB) * 1024 * 1024
|
||||
allowedCollections := wildcard.CompileWildcardMatchers(taskConfig.CollectionFilter)
|
||||
allowedCollections, err := pluginworker.CompileCollectionMatcher(taskConfig.CollectionFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
volumeGroups := make(map[uint32][]*workertypes.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
@@ -334,7 +336,7 @@ func emitErasureCodingDetectionDecisionTrace(
|
||||
skippedTooSmall++
|
||||
continue
|
||||
}
|
||||
if len(allowedCollections) > 0 && !wildcard.MatchesAnyWildcard(allowedCollections, metric.Collection) {
|
||||
if !allowedCollections.Matches(metric.Collection) {
|
||||
skippedCollectionFilter++
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user