mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
fix(ec): fence stale-worker EC shard cleanup by encode generation (#9953)
* feat(ec): add encode_ts_ns to the EC task params, shard-unmount, and shard-delete RPCs The generation fence for stale EC-worker cleanup needs the encode generation on three messages: ErasureCodingTaskParams (admin issues it), VolumeEcShardsUnmountRequest, and VolumeEcShardsDeleteRequest (the worker carries it to the volume server). Additive fields only; 0 preserves the existing unfenced behavior. Mirror the two volume-server fields in the Rust volume server's proto copy. * feat(ec): issue the EC encode generation from the admin and carry it on the worker Stamp each EC proposal's encode_ts_ns from the admin's per-cycle DetectionSequence (a single-clock value) so generations are globally ordered even though detection runs on a rotating worker. The worker writes that generation into the distributed .vif and passes it on its shard unmount/delete RPCs; it falls back to a local timestamp for the .vif only on the unfenced legacy/shell path (keeping the read guard on). * fix(ec): fence the stale-worker EC shard unmount and teardown by generation A reaped-but-still-running EC worker's cleanupStaleEcShards issued a generation-blind unmount + full teardown that could unmount and then overwrite a newer run's live shards on a shared node. Both RPCs now carry the encode generation: the volume server unmounts/deletes a disk only when its .vif generation is strictly older than the request, and preserves a same-or-newer generation, a generation-0 (recovered or pre-upgrade) volume, and an unreadable .vif. Unload is per-disk, never node-wide. Request generation 0 keeps the blanket teardown for the shell pre-encode cleanup and pre-upgrade callers. Mirrored in the Rust volume server. * test(ec): cover the generation-fenced teardown and unmount End-to-end volume-server tests: a fenced FullTeardown wipes a strictly- older generation, preserves a newer one, preserves a generation-0 volume, and blanket-wipes on request generation 0; the gen-aware unmount preserves a same-or-newer mounted generation; and the .vif generation reader handles present/absent/no-config cases. * test(ec): pin the fenced .vif==teardown generation and the unreadable-.vif preserve A fenced run must stamp the admin generation verbatim into the .vif so it matches the generation sent on the teardown RPCs; add a regression test that sets the task generation and asserts the .vif carries it exactly. Also cover the present-but-unparseable .vif case (reads as generation 0, preserved) and correct the readEcGenerationTsNs docstring accordingly. * fix(ec): surface EC full-teardown filesystem errors in the Rust volume server remove_ec_volume_files(_full_teardown) discarded every fs::remove_file error, so a teardown that failed on permissions or a full disk still returned full_teardown_done=true and left stale artifacts to collide with the next encode. Return io::Result, ignore NotFound, propagate the first real error, and have the teardown RPC surface it -- matching the Go contract. The best-effort reconcile/load-cleanup callers keep ignoring it. * refactor(ec): reuse the EC volume lookup on unmount and short-circuit the gen read Address review: the Rust unmount fence reuses the ec_vol it already fetched instead of a second find_ec_volume; the Go .vif generation reader breaks out of the data/idx loop early when the two dirs are the same.
This commit is contained in:
1 parent
561768a426
commit
284796c7b6
16 files changed
+549
-49
No files matched your search
@@ -2662,15 +2662,54 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
if req.full_teardown {
|
||||
// Pre-encode cleanup: evict the volume and wipe every EC artifact for it
|
||||
// on every disk, not just the listed shards, so a remote node retains no
|
||||
// stale generation that a fresh gen-0 copy would collide with. Echo the
|
||||
// acknowledgement so the caller can tell a pre-upgrade server apart.
|
||||
{
|
||||
if req.encode_ts_ns == 0 {
|
||||
// Blanket teardown (shell pre-encode cleanup / pre-upgrade caller): evict
|
||||
// the volume and wipe every EC artifact for it on every disk, not just the
|
||||
// listed shards, so a remote node retains no stale generation a fresh copy
|
||||
// collides with. Echo the acknowledgement so the caller can tell a
|
||||
// pre-upgrade server apart.
|
||||
{
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
let _ = store.remove_ec_volume(vid);
|
||||
for loc in &store.locations {
|
||||
loc.remove_ec_volume_files_full_teardown(&req.collection, vid)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!(
|
||||
"full teardown of ec volume {}: {}",
|
||||
req.volume_id, e
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Generation-fenced teardown (stale-worker pre-distribute cleanup): wipe
|
||||
// only a disk whose .vif generation is strictly OLDER than the request;
|
||||
// preserve same-or-newer, generation 0 (recovered/pre-upgrade live volume),
|
||||
// and an unreadable .vif, so a stale run never wipes a newer run's live
|
||||
// shards. Unload and remove only the strictly-older disks, never node-wide.
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
let _ = store.remove_ec_volume(vid);
|
||||
for loc in &store.locations {
|
||||
loc.remove_ec_volume_files_full_teardown(&req.collection, vid);
|
||||
for disk_id in 0..store.locations.len() {
|
||||
let disk_gen = store.locations[disk_id].ec_generation_ts_ns(&req.collection, vid);
|
||||
let older = matches!(disk_gen, Some(g) if g > 0 && g < req.encode_ts_ns);
|
||||
if !older {
|
||||
tracing::info!(
|
||||
volume_id = vid.0,
|
||||
disk_id,
|
||||
?disk_gen,
|
||||
req = req.encode_ts_ns,
|
||||
"ec full teardown preserved: generation not older than request"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
store.locations[disk_id].remove_ec_volume(vid);
|
||||
store.locations[disk_id]
|
||||
.remove_ec_volume_files_full_teardown(&req.collection, vid)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!(
|
||||
"fenced teardown of ec volume {}: {}",
|
||||
req.volume_id, e
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
self.state.volume_state_notify.notify_one();
|
||||
@@ -2728,9 +2767,11 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Matches Go: for _, shardId := range req.ShardIds { err = vs.store.UnmountEcShards(...) }
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
for &shard_id in &req.shard_ids {
|
||||
store.unmount_ec_shard(vid, shard_id).map_err(|e| {
|
||||
Status::internal(format!("unmount {}.{}: {}", req.volume_id, shard_id, e))
|
||||
})?;
|
||||
store
|
||||
.unmount_ec_shard(vid, shard_id, req.encode_ts_ns)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!("unmount {}.{}: {}", req.volume_id, shard_id, e))
|
||||
})?;
|
||||
}
|
||||
drop(store);
|
||||
self.state.volume_state_notify.notify_one();
|
||||
|
||||
@@ -157,7 +157,7 @@ impl DiskLocation {
|
||||
volume_id = vid.0,
|
||||
"EC volume validation failed, removing incomplete EC files"
|
||||
);
|
||||
self.remove_ec_volume_files(&collection, vid);
|
||||
let _ = self.remove_ec_volume_files(&collection, vid);
|
||||
// Fall through to load .dat file
|
||||
}
|
||||
}
|
||||
@@ -377,24 +377,25 @@ impl DiskLocation {
|
||||
/// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
|
||||
/// when a healthy `.dat` for the same vid lives on a sibling disk
|
||||
/// (seaweedfs/seaweedfs#9478).
|
||||
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) -> io::Result<()> {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
const MAX_SHARD_COUNT: usize = 32;
|
||||
|
||||
// Remove index files from idx directory (.ecx, .ecj)
|
||||
let _ = fs::remove_file(format!("{}.ecx", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", idx_base));
|
||||
rm_if_present(format!("{}.ecx", idx_base))?;
|
||||
rm_if_present(format!("{}.ecj", idx_base))?;
|
||||
// Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
|
||||
if self.idx_directory != self.directory {
|
||||
let _ = fs::remove_file(format!("{}.ecx", base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", base));
|
||||
rm_if_present(format!("{}.ecx", base))?;
|
||||
rm_if_present(format!("{}.ecj", base))?;
|
||||
}
|
||||
|
||||
// Remove all EC shard files (.ec00 ~ .ec31)
|
||||
for i in 0..MAX_SHARD_COUNT {
|
||||
let _ = fs::remove_file(format!("{}.ec{:02}", base, i));
|
||||
rm_if_present(format!("{}.ec{:02}", base, i))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full-teardown variant: everything remove_ec_volume_files clears, PLUS the
|
||||
@@ -405,8 +406,12 @@ impl DiskLocation {
|
||||
/// .vif. Reconcile/load-fallback call remove_ec_volume_files directly and
|
||||
/// intentionally preserve it, mirroring Go's removeEcVolumeFiles (reconcile) vs
|
||||
/// removeStaleEcArtifacts (teardown) split.
|
||||
pub(crate) fn remove_ec_volume_files_full_teardown(&self, collection: &str, vid: VolumeId) {
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
pub(crate) fn remove_ec_volume_files_full_teardown(
|
||||
&self,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<()> {
|
||||
self.remove_ec_volume_files(collection, vid)?;
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
// try_exists, not exists(): a stat error on a PRESENT .idx must not read as
|
||||
@@ -416,16 +421,36 @@ impl DiskLocation {
|
||||
std::path::Path::new(&format!("{}.idx", idx_base)).try_exists(),
|
||||
Ok(false)
|
||||
) {
|
||||
let _ = fs::remove_file(format!("{}.vif", idx_base));
|
||||
rm_if_present(format!("{}.vif", idx_base))?;
|
||||
if self.idx_directory != self.directory
|
||||
&& matches!(
|
||||
std::path::Path::new(&format!("{}.idx", base)).try_exists(),
|
||||
Ok(false)
|
||||
)
|
||||
{
|
||||
let _ = fs::remove_file(format!("{}.vif", base));
|
||||
rm_if_present(format!("{}.vif", base))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// EC encode generation recorded in this disk's .vif (data dir first, then idx
|
||||
/// dir for the split-disk layout). None if no .vif was readable, so the fenced
|
||||
/// teardown preserves it (fail-safe). A present .vif with no ec_shard_config
|
||||
/// yields Some(0) (a recovered or pre-upgrade live volume), also preserved.
|
||||
pub(crate) fn ec_generation_ts_ns(&self, collection: &str, vid: VolumeId) -> Option<i64> {
|
||||
for dir in [&self.directory, &self.idx_directory] {
|
||||
let vif = format!("{}.vif", volume_file_name(dir, collection, vid));
|
||||
if let Ok(s) = fs::read_to_string(&vif) {
|
||||
if let Ok(vi) = serde_json::from_str::<VifVolumeInfo>(&s) {
|
||||
return Some(vi.ec_shard_config.map(|c| c.encode_ts_ns).unwrap_or(0));
|
||||
}
|
||||
}
|
||||
if self.directory == self.idx_directory {
|
||||
break;
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find a volume by ID.
|
||||
@@ -924,7 +949,7 @@ impl DiskLocation {
|
||||
volume_id = vid.0,
|
||||
"Incomplete or invalid EC volume: .dat exists but validation failed, cleaning up EC files",
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
let _ = self.remove_ec_volume_files(collection, vid);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -966,7 +991,7 @@ impl DiskLocation {
|
||||
"Found {} EC shards without .ecx file (incomplete encoding interrupted before .ecx), cleaning up",
|
||||
shards.len(),
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
let _ = self.remove_ec_volume_files(collection, vid);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
@@ -1046,6 +1071,16 @@ fn calculate_expected_shard_size(dat_file_size: i64, data_shards: usize) -> i64
|
||||
/// Resolve the EC data-shard count from the volume's own `.vif` (the volume
|
||||
/// server never holds the cluster EC config in memory), checking the data dir
|
||||
/// then the idx dir. Falls back to the default ratio when no EC `.vif` is found.
|
||||
/// Remove a file, treating a missing file as success but propagating real errors
|
||||
/// (permissions, disk full). Mirrors Go's removeFileIfExists so a teardown failure
|
||||
/// surfaces instead of silently leaving stale artifacts.
|
||||
fn rm_if_present(path: String) -> io::Result<()> {
|
||||
match fs::remove_file(&path) {
|
||||
Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn ec_data_shards_from_vif(directory: &str, idx_directory: &str, collection: &str, vid: VolumeId) -> usize {
|
||||
for dir in [directory, idx_directory] {
|
||||
let vif = format!("{}.vif", volume_file_name(dir, collection, vid));
|
||||
@@ -1278,6 +1313,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The per-disk generation reader behind the fenced EC teardown: a present .vif
|
||||
// yields its generation (or 0 with no EC config), a missing .vif is unreadable
|
||||
// (preserved, fail-safe), and the idx dir is a fallback for the split-disk layout.
|
||||
#[test]
|
||||
fn test_ec_generation_ts_ns_reads_and_fences() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let data = tmp.path().join("data");
|
||||
let idx = tmp.path().join("idx");
|
||||
std::fs::create_dir_all(&data).unwrap();
|
||||
std::fs::create_dir_all(&idx).unwrap();
|
||||
let loc = DiskLocation::new(
|
||||
data.to_str().unwrap(),
|
||||
idx.to_str().unwrap(),
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let vid = VolumeId(88);
|
||||
let dbase = volume_file_name(data.to_str().unwrap(), "", vid);
|
||||
let ibase = volume_file_name(idx.to_str().unwrap(), "", vid);
|
||||
|
||||
// No .vif anywhere -> unreadable (the fenced teardown preserves on None).
|
||||
assert_eq!(loc.ec_generation_ts_ns("", vid), None);
|
||||
|
||||
// .vif in the data dir carries its generation.
|
||||
let with_gen = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 4242,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
std::fs::write(format!("{}.vif", dbase), serde_json::to_string(&with_gen).unwrap()).unwrap();
|
||||
assert_eq!(loc.ec_generation_ts_ns("", vid), Some(4242));
|
||||
|
||||
// A .vif with no EC config reads as generation 0 (recovered/pre-upgrade live volume).
|
||||
std::fs::remove_file(format!("{}.vif", dbase)).unwrap();
|
||||
let no_cfg = VifVolumeInfo {
|
||||
version: 3,
|
||||
..Default::default()
|
||||
};
|
||||
std::fs::write(format!("{}.vif", dbase), serde_json::to_string(&no_cfg).unwrap()).unwrap();
|
||||
assert_eq!(loc.ec_generation_ts_ns("", vid), Some(0));
|
||||
|
||||
// idx-dir fallback: only the idx dir holds the .vif.
|
||||
std::fs::remove_file(format!("{}.vif", dbase)).unwrap();
|
||||
std::fs::write(format!("{}.vif", ibase), serde_json::to_string(&with_gen).unwrap()).unwrap();
|
||||
assert_eq!(loc.ec_generation_ts_ns("", vid), Some(4242));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_volume_filename() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -716,17 +716,37 @@ impl Store {
|
||||
|
||||
/// Unmount a single EC shard, searching all locations.
|
||||
/// Matches Go's Store.UnmountEcShards which unmounts one shard at a time.
|
||||
pub fn unmount_ec_shard(&mut self, vid: VolumeId, shard_id: u32) -> Result<(), VolumeError> {
|
||||
pub fn unmount_ec_shard(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
shard_id: u32,
|
||||
req_encode_ts_ns: i64,
|
||||
) -> Result<(), VolumeError> {
|
||||
// Walk all locations rather than stopping at the first with the
|
||||
// vid — split-disk reconciled volumes can have the same vid on
|
||||
// multiple disks, with the target shard on any of them.
|
||||
for disk_id in 0..self.locations.len() {
|
||||
let has_shard = self.locations[disk_id]
|
||||
.find_ec_volume(vid)
|
||||
.is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
|
||||
let ec_vol = self.locations[disk_id].find_ec_volume(vid);
|
||||
let has_shard = ec_vol.is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
|
||||
if !has_shard {
|
||||
continue;
|
||||
}
|
||||
// Generation fence: when the caller carries a generation (stale-worker
|
||||
// cleanup), only unmount a strictly-older generation; preserve a disk
|
||||
// whose generation is same-or-newer, 0, or unknown, so a stale run cannot
|
||||
// unmount a newer run's live shards. req 0 (legacy/shell) unmounts all.
|
||||
let disk_gen = ec_vol.map_or(0, |v| v.encode_ts_ns);
|
||||
if req_encode_ts_ns > 0 && !(disk_gen > 0 && disk_gen < req_encode_ts_ns) {
|
||||
tracing::info!(
|
||||
volume_id = vid.0,
|
||||
shard_id,
|
||||
disk_id,
|
||||
disk_gen,
|
||||
req = req_encode_ts_ns,
|
||||
"UnmountEcShards skipped: generation not older than request"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
tracing::info!(
|
||||
volume_id = vid.0,
|
||||
shard_id,
|
||||
|
||||
@@ -332,7 +332,7 @@ impl Store {
|
||||
// Also sweep any unmounted shard files (.ec00 .. .ec31)
|
||||
// that the per-disk loader skipped — destroy() only walks
|
||||
// the in-memory shards, but the disk may still hold others.
|
||||
loc.remove_ec_volume_files(&v.collection, v.vid);
|
||||
let _ = loc.remove_ec_volume_files(&v.collection, v.vid);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1044,7 +1044,7 @@ mod tests {
|
||||
let (mut store, _tmp) = build_split_disk_store(7003);
|
||||
let vid = VolumeId(7003);
|
||||
|
||||
store.unmount_ec_shard(vid, 1).unwrap();
|
||||
store.unmount_ec_shard(vid, 1, 0).unwrap();
|
||||
assert_eq!(store.find_ec_shard_location(vid, 1), None);
|
||||
// Other shards untouched.
|
||||
assert_eq!(store.find_ec_shard_location(vid, 0), Some(0));
|
||||
|
||||
Reference in new issue
Block a user