From 8c8d0f1cdf0328162b1f3f4f86d6a2b6d1d31b55 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:31:25 +0530 Subject: [PATCH] weed/shell: resolve collection from topology when ec.decode is given only volumeId (#12424) * weed/shell: resolve collection from topology when ec.decode is given only volumeId When ec.decode is invoked with -volumeId for a volume that belongs to a named collection without explicitly specifying -collection, DoEcDecode was called with an empty collection string. Target volume servers then rejected VolumeEcShardsCopy with 'CopyFile not found ec volume id ' because they looked under the default collection. This commit introduces FindEcVolumeCollection to discover the collection name directly from topology metadata (checking EC shard info and regular volume info), ensuring DoEcDecode receives the accurate collection. * ec.decode: resolve every batch volume's collection from the topology The batch passed -collection through unless it looked like a regex, so a comma list (a,b), a {n} quantifier, ALL or _default_ reached the volume servers as a collection name and the copy found no shards. The selector only picks the volumes; each one is decoded in its own collection. Co-Authored-By: Claude Opus 5.5 * ec.decode: take the collection only from the volume's ec shards The lookup returned the first record per disk, so a regular volume on an earlier node answered before the ec shards on a later one. A decode needs the shards anyway, so their records are the only ones to read. Co-Authored-By: Claude Opus 5.5 * ec.decode: leave the -volumeId lookup to DoEcDecode DoEcDecode already resolves an empty collection from the topology. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- weed/ec/ec_decode.go | 33 +++++++++++ weed/ec/ec_decode_test.go | 98 +++++++++++++++++++++++++++++++++ weed/shell/command_ec_decode.go | 4 +- 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/weed/ec/ec_decode.go b/weed/ec/ec_decode.go index 985b522c5d..b0c3c2ef6c 100644 --- a/weed/ec/ec_decode.go +++ b/weed/ec/ec_decode.go @@ -26,6 +26,12 @@ func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, v return fmt.Errorf("lock is lost") } + if collection == "" { + if c, found := FindEcVolumeCollection(topoInfo, vid); found { + collection = c + } + } + // find volume location nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid) @@ -376,6 +382,33 @@ func CollectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionMatcher *wild return } +// FindEcVolumeCollection returns the collection of the ec volume's shards. +func FindEcVolumeCollection(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (string, bool) { + if topoInfo == nil { + return "", false + } + var collection string + var found bool + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if found { + return + } + for _, diskInfo := range dn.DiskInfos { + if diskInfo == nil { + continue + } + for _, v := range diskInfo.EcShardInfos { + if v.Id == uint32(vid) { + collection = v.Collection + found = true + return + } + } + } + }) + return collection, found +} + func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) { res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo) EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { diff --git a/weed/ec/ec_decode_test.go b/weed/ec/ec_decode_test.go index bf73126e86..ae69a11fdb 100644 --- a/weed/ec/ec_decode_test.go +++ b/weed/ec/ec_decode_test.go @@ -4,7 +4,9 @@ import ( "testing" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) // shardsOn builds one holder's inventory. @@ -89,3 +91,99 @@ func TestMissingDataShard(t *testing.T) { }) } } + +func TestFindEcVolumeCollection(t *testing.T) { + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + Id: "dc1", + RackInfos: []*master_pb.RackInfo{ + { + Id: "rack1", + DataNodeInfos: []*master_pb.DataNodeInfo{ + { + Id: "node1", + DiskInfos: map[string]*master_pb.DiskInfo{ + "hdd": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 10, + Collection: "s3bucket", + }, + { + Id: 30, + Collection: "", + }, + }, + VolumeInfos: []*master_pb.VolumeInformationMessage{ + { + Id: 20, + Collection: "regular_col", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + topo *master_pb.TopologyInfo + vid needle.VolumeId + wantCol string + wantFound bool + }{ + { + name: "nil topology", + topo: nil, + vid: 10, + wantCol: "", + wantFound: false, + }, + { + name: "ec volume with named collection", + topo: topo, + vid: 10, + wantCol: "s3bucket", + wantFound: true, + }, + { + name: "ec volume with empty collection", + topo: topo, + vid: 30, + wantCol: "", + wantFound: true, + }, + { + name: "regular volume record is not an ec volume", + topo: topo, + vid: 20, + wantCol: "", + wantFound: false, + }, + { + name: "not found", + topo: topo, + vid: 999, + wantCol: "", + wantFound: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + col, found := FindEcVolumeCollection(tt.topo, tt.vid) + if found != tt.wantFound { + t.Fatalf("FindEcVolumeCollection() found = %v, want %v", found, tt.wantFound) + } + if col != tt.wantCol { + t.Errorf("FindEcVolumeCollection() col = %q, want %q", col, tt.wantCol) + } + }) + } +} diff --git a/weed/shell/command_ec_decode.go b/weed/shell/command_ec_decode.go index 33e640d182..a33a46c4af 100644 --- a/weed/shell/command_ec_decode.go +++ b/weed/shell/command_ec_decode.go @@ -113,7 +113,9 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr fmt.Printf("ec decode batch %d/%d: %v\n", i+1, len(batches), batch) } for _, vid := range batch { - if err = ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil { + // -collection is a selector, not the volume's collection + volumeCollection, _ := ec.FindEcVolumeCollection(topologyInfo, vid) + if err = ec.DoEcDecode(env, topologyInfo, volumeCollection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil { return err } }