mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
fix(ec): fence stale-worker EC shard cleanup by encode generation (#9953)
* feat(ec): add encode_ts_ns to the EC task params, shard-unmount, and shard-delete RPCs The generation fence for stale EC-worker cleanup needs the encode generation on three messages: ErasureCodingTaskParams (admin issues it), VolumeEcShardsUnmountRequest, and VolumeEcShardsDeleteRequest (the worker carries it to the volume server). Additive fields only; 0 preserves the existing unfenced behavior. Mirror the two volume-server fields in the Rust volume server's proto copy. * feat(ec): issue the EC encode generation from the admin and carry it on the worker Stamp each EC proposal's encode_ts_ns from the admin's per-cycle DetectionSequence (a single-clock value) so generations are globally ordered even though detection runs on a rotating worker. The worker writes that generation into the distributed .vif and passes it on its shard unmount/delete RPCs; it falls back to a local timestamp for the .vif only on the unfenced legacy/shell path (keeping the read guard on). * fix(ec): fence the stale-worker EC shard unmount and teardown by generation A reaped-but-still-running EC worker's cleanupStaleEcShards issued a generation-blind unmount + full teardown that could unmount and then overwrite a newer run's live shards on a shared node. Both RPCs now carry the encode generation: the volume server unmounts/deletes a disk only when its .vif generation is strictly older than the request, and preserves a same-or-newer generation, a generation-0 (recovered or pre-upgrade) volume, and an unreadable .vif. Unload is per-disk, never node-wide. Request generation 0 keeps the blanket teardown for the shell pre-encode cleanup and pre-upgrade callers. Mirrored in the Rust volume server. * test(ec): cover the generation-fenced teardown and unmount End-to-end volume-server tests: a fenced FullTeardown wipes a strictly- older generation, preserves a newer one, preserves a generation-0 volume, and blanket-wipes on request generation 0; the gen-aware unmount preserves a same-or-newer mounted generation; and the .vif generation reader handles present/absent/no-config cases. * test(ec): pin the fenced .vif==teardown generation and the unreadable-.vif preserve A fenced run must stamp the admin generation verbatim into the .vif so it matches the generation sent on the teardown RPCs; add a regression test that sets the task generation and asserts the .vif carries it exactly. Also cover the present-but-unparseable .vif case (reads as generation 0, preserved) and correct the readEcGenerationTsNs docstring accordingly. * fix(ec): surface EC full-teardown filesystem errors in the Rust volume server remove_ec_volume_files(_full_teardown) discarded every fs::remove_file error, so a teardown that failed on permissions or a full disk still returned full_teardown_done=true and left stale artifacts to collide with the next encode. Return io::Result, ignore NotFound, propagate the first real error, and have the teardown RPC surface it -- matching the Go contract. The best-effort reconcile/load-cleanup callers keep ignoring it. * refactor(ec): reuse the EC volume lookup on unmount and short-circuit the gen read Address review: the Rust unmount fence reuses the ec_vol it already fetched instead of a second find_ec_volume; the Go .vif generation reader breaks out of the data/idx loop early when the two dirs are the same.
This commit is contained in:
1 parent
561768a426
commit
284796c7b6
16 files changed
+549
-49
No files matched your search
@@ -3606,6 +3606,7 @@ type VolumeEcShardsDeleteRequest struct {
|
||||
Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
|
||||
ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"`
|
||||
FullTeardown bool `protobuf:"varint,4,opt,name=full_teardown,json=fullTeardown,proto3" json:"full_teardown,omitempty"` // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
|
||||
EncodeTsNs int64 `protobuf:"varint,5,opt,name=encode_ts_ns,json=encodeTsNs,proto3" json:"encode_ts_ns,omitempty"` // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -3668,6 +3669,13 @@ func (x *VolumeEcShardsDeleteRequest) GetFullTeardown() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *VolumeEcShardsDeleteRequest) GetEncodeTsNs() int64 {
|
||||
if x != nil {
|
||||
return x.EncodeTsNs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type VolumeEcShardsDeleteResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
FullTeardownDone bool `protobuf:"varint,1,opt,name=full_teardown_done,json=fullTeardownDone,proto3" json:"full_teardown_done,omitempty"` // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op
|
||||
@@ -3820,6 +3828,7 @@ type VolumeEcShardsUnmountRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
|
||||
ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"`
|
||||
EncodeTsNs int64 `protobuf:"varint,4,opt,name=encode_ts_ns,json=encodeTsNs,proto3" json:"encode_ts_ns,omitempty"` // generation fence: skip a disk whose mounted EC volume is this generation or newer (0 = unfenced, unmount all)
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -3868,6 +3877,13 @@ func (x *VolumeEcShardsUnmountRequest) GetShardIds() []uint32 {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *VolumeEcShardsUnmountRequest) GetEncodeTsNs() int64 {
|
||||
if x != nil {
|
||||
return x.EncodeTsNs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type VolumeEcShardsUnmountResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
@@ -7268,14 +7284,16 @@ const file_volume_server_proto_rawDesc = "" +
|
||||
"\rcopy_vif_file\x18\a \x01(\bR\vcopyVifFile\x12\x17\n" +
|
||||
"\adisk_id\x18\b \x01(\rR\x06diskId\x12&\n" +
|
||||
"\x0fcopy_ecsum_file\x18\t \x01(\bR\rcopyEcsumFile\"\x1c\n" +
|
||||
"\x1aVolumeEcShardsCopyResponse\"\x9c\x01\n" +
|
||||
"\x1aVolumeEcShardsCopyResponse\"\xbe\x01\n" +
|
||||
"\x1bVolumeEcShardsDeleteRequest\x12\x1b\n" +
|
||||
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" +
|
||||
"\n" +
|
||||
"collection\x18\x02 \x01(\tR\n" +
|
||||
"collection\x12\x1b\n" +
|
||||
"\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12#\n" +
|
||||
"\rfull_teardown\x18\x04 \x01(\bR\ffullTeardown\"L\n" +
|
||||
"\rfull_teardown\x18\x04 \x01(\bR\ffullTeardown\x12 \n" +
|
||||
"\fencode_ts_ns\x18\x05 \x01(\x03R\n" +
|
||||
"encodeTsNs\"L\n" +
|
||||
"\x1cVolumeEcShardsDeleteResponse\x12,\n" +
|
||||
"\x12full_teardown_done\x18\x01 \x01(\bR\x10fullTeardownDone\"\xa0\x01\n" +
|
||||
"\x1aVolumeEcShardsMountRequest\x12\x1b\n" +
|
||||
@@ -7285,10 +7303,12 @@ const file_volume_server_proto_rawDesc = "" +
|
||||
"collection\x12\x1b\n" +
|
||||
"\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12(\n" +
|
||||
"\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\"\x1d\n" +
|
||||
"\x1bVolumeEcShardsMountResponse\"X\n" +
|
||||
"\x1bVolumeEcShardsMountResponse\"z\n" +
|
||||
"\x1cVolumeEcShardsUnmountRequest\x12\x1b\n" +
|
||||
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1b\n" +
|
||||
"\tshard_ids\x18\x03 \x03(\rR\bshardIds\"\x1f\n" +
|
||||
"\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12 \n" +
|
||||
"\fencode_ts_ns\x18\x04 \x01(\x03R\n" +
|
||||
"encodeTsNs\"\x1f\n" +
|
||||
"\x1dVolumeEcShardsUnmountResponse\"\xc1\x01\n" +
|
||||
"\x18VolumeEcShardReadRequest\x12\x1b\n" +
|
||||
"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x19\n" +
|
||||
|
||||
Reference in new issue
Block a user