From 3159b8408c2dafc9bf316106dc51434a99830c7c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 19 Nov 2026 00:02:43 -0800 Subject: [PATCH] filer, volume: fsync chunk deletes under an fsync path rule (#12759) * volume: fsync a batch delete that asks for it BatchDeleteRequest gets an fsync field, and the Go and Rust volume servers pass it to the needle delete, which already orders the tombstone, .dat sync, publish and .idx sync for a durable delete. Co-Authored-By: Claude Opus 5.5 * operation: pass an fsync flag to batch deletes DeleteFileIdsWithLookupVolumeId and DeleteFileIdsAtOneVolumeServer take an fsync argument and send it in the BatchDeleteRequest. Every caller passes false for now. Co-Authored-By: Claude Opus 5.5 * filer: fsync chunk deletes under an fsync path rule DeleteChunks passes the path rule's fsync flag to the deletion queue. The flag is kept with the pending record until the delete is confirmed, so retries and re-queues keep it, and the batch and retry loops send marked ids in a durable BatchDelete. Co-Authored-By: Claude Opus 5.5 * filer: keep fsync marks in the deletion ledger Marked ids go into the trailing parts of a manifest ledger, counted by fsyncParts. Older filers read those parts as plain ids and still delete them; a reload or a claimed foreign ledger restores the marks. A ledger without marks keeps the single array form. Co-Authored-By: Claude Opus 5.5 * filer: record a queued deletion before the delete loop can see it queueDeletions published the ids to the hot queue before taking the ledger lock, so a concurrent flush could delete an id before its fsync mark landed and then forget the mark with the record. Co-Authored-By: Claude Opus 5.5 * filer: fsync a folder delete's chunks by each child's path rule A recursive delete collected its file children's chunks and resolved the rule once for the folder, which misses a trailing-slash rule on the folder itself and file-prefix rules. Each file child's own rule now decides fsync; the folder rule still decides chunk deletion. Co-Authored-By: Claude Opus 5.5 * filer: keep a newer fsync request when a plain delete finishes An id re-queued with fsync while a plain delete of it was in flight lost its mark when that delete succeeded, so the queued copy also ran without fsync. A plain attempt now leaves a marked record for its queued copy; a durable delete of an already-deleted needle still syncs the tombstone. Co-Authored-By: Claude Opus 5.5 * filer: delete a folder's plain child chunks without fsync Children whose own rule turns fsync off were still deleted durably when the folder's rule asked for fsync. Only the children marked by their own rule now go out with fsync. Co-Authored-By: Claude Opus 5.5 * filer: retry a durable chunk delete the volume server failed A volume server rolls back a durable delete it cannot sync and answers with a 5xx carrying the I/O error text, which the classifier took as permanent, so the record and its fsync mark were dropped while the needle stayed live. A 5xx answer to a durable delete is now retryable, within the usual attempt limit. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- seaweed-volume/proto/volume_server.proto | 1 + seaweed-volume/src/server/grpc_server.rs | 56 ++- seaweed-volume/src/server/handlers.rs | 1 + weed/filer/filer.go | 4 +- weed/filer/filer_delete_entry.go | 16 +- weed/filer/filer_deletion.go | 70 ++-- weed/filer/filer_deletion_fsync_test.go | 396 +++++++++++++++++++ weed/filer/filer_deletion_persist.go | 176 ++++++--- weed/filer/filer_deletion_persist_test.go | 44 +-- weed/operation/delete_content.go | 10 +- weed/pb/volume_server.proto | 1 + weed/pb/volume_server_pb/volume_server.pb.go | 13 +- weed/s3api/s3api_object_handlers_put.go | 2 +- weed/server/volume_grpc_batch_delete.go | 2 +- weed/server/volume_grpc_batch_delete_test.go | 53 +++ weed/shell/command_fs_merge_volumes.go | 2 +- weed/shell/command_volume_check_disk.go | 2 +- weed/shell/command_volume_fsck.go | 2 +- 18 files changed, 743 insertions(+), 108 deletions(-) create mode 100644 weed/filer/filer_deletion_fsync_test.go diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto index b36932c9eb..2faea41a85 100644 --- a/seaweed-volume/proto/volume_server.proto +++ b/seaweed-volume/proto/volume_server.proto @@ -147,6 +147,7 @@ service VolumeServer { message BatchDeleteRequest { repeated string file_ids = 1; bool skip_cookie_check = 2; + bool fsync = 3; } message BatchDeleteResponse { diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 23eb10d2db..9d263dce40 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -1276,7 +1276,7 @@ impl VolumeServer for VolumeGrpcService { Some(vol) if vol.is_no_write_or_delete() => { Err(crate::storage::volume::VolumeError::ReadOnly(file_id.volume_id)) } - Some(vol) => vol.delete_needle(&mut n, false), + Some(vol) => vol.delete_needle(&mut n, req.fsync), }; match delete_result { Ok(size) => { @@ -14237,6 +14237,7 @@ mod tests { .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec![fid.to_string()], skip_cookie_check, + fsync: false, })) .await .unwrap() @@ -14290,6 +14291,57 @@ mod tests { ); } + /// Go's TestBatchDeleteFsyncsOnlyWhenAsked. + #[tokio::test] + async fn test_batch_delete_fsyncs_only_when_asked() { + let (service, _tmp) = make_local_service_with_volume("", None); + for (id, fsync) in [(0x6e7a_0c01u64, false), (0x6e7a_0c02, true)] { + let mut n = Needle { + id: NeedleId(id), + cookie: Cookie(0x3344), + data: b"chunk".to_vec(), + data_size: 5, + ..Needle::default() + }; + service + .state + .store + .write() + .unwrap() + .write_volume_needle(VolumeId(1), &mut n, false) + .unwrap(); + let syncs = || { + let store = service.state.store.read().unwrap(); + let (_, vol) = store.find_volume(VolumeId(1)).unwrap(); + vol.sync_counts_for_test() + }; + let before = syncs(); + let fid = needle::FileId::new(VolumeId(1), NeedleId(id), Cookie(0x3344)); + let resp = service + .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { + file_ids: vec![fid.to_string()], + skip_cookie_check: true, + fsync, + })) + .await + .unwrap() + .into_inner(); + assert_eq!(resp.results[0].status, 202, "{:?}", resp.results); + let after = syncs(); + if fsync { + assert!( + after.0 > before.0 && after.1 > before.1, + "the tombstone was not synced: {before:?} -> {after:?}" + ); + } else { + assert_eq!( + after, before, + "a delete that did not ask for fsync paid for one" + ); + } + } + } + const GO_EC_NEEDLE_NOT_FOUND: &str = "locate in local ec volume: FindNeedleFromEcx: needle not found"; @@ -16327,6 +16379,7 @@ mod tests { Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec!["1,b00003344".to_string()], skip_cookie_check: false, + fsync: false, }) }; @@ -16376,6 +16429,7 @@ mod tests { .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec!["2,b00003344".to_string()], skip_cookie_check, + fsync: false, })) .await .unwrap() diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 73af9c0058..53b5ad9194 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -683,6 +683,7 @@ async fn batch_delete_file_ids( .batch_delete(volume_server_pb::BatchDeleteRequest { file_ids: batch.clone(), skip_cookie_check: true, + fsync: false, }) .await .map_err(|e| format!("batch delete {}: {}", grpc_addr, e))? diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 1b9b10664f..48c8dd7631 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -107,9 +107,11 @@ type Filer struct { // to the store so a restart does not leak chunks. Guarded by // deletionLedgerLock; nil-safe for Filer literals in tests. // pendingDeletions maps each id to its enqueue epoch so an expiry-forget - // cannot erase a re-queued id. + // cannot erase a re-queued id. fsyncDeletions marks pending ids whose + // volume deletes must be durable. deletionLedgerLock sync.Mutex pendingDeletions map[string]uint64 + fsyncDeletions map[string]struct{} deletionSeq uint64 deletionLedgerDirty bool deletionLedgerParts int diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 2d82d532f0..e8a0e076cb 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -205,7 +205,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry } //collect all the chunks of this layer and delete them together at the end - var chunksToDelete []*filer_pb.FileChunk + var chunksToDelete, fsyncChunksToDelete []*filer_pb.FileChunk lastFileName := "" includeLastFile := false listedChildren := !isDeletingBucket || !f.Store.CanDropWholeBucket() @@ -246,15 +246,19 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry break } f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, sub), nil, shouldDeleteChunks, isFromOtherCluster, nil) + subChunksToDelete := &chunksToDelete + if shouldDeleteChunks && f.FilerConf.MatchStorageRule(string(sub.FullPath)).GetFsync() { + subChunksToDelete = &fsyncChunksToDelete + } if len(sub.HardLinkId) != 0 { // hard link chunk data are deleted separately err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId}) if shouldDeleteChunks && f.hardLinkIsOrphaned(ctx, sub.HardLinkId) { - chunksToDelete = append(chunksToDelete, sub.GetChunks()...) + *subChunksToDelete = append(*subChunksToDelete, sub.GetChunks()...) } } else { if shouldDeleteChunks { - chunksToDelete = append(chunksToDelete, sub.GetChunks()...) + *subChunksToDelete = append(*subChunksToDelete, sub.GetChunks()...) } } } @@ -280,7 +284,11 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry } f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, entry), nil, shouldDeleteChunks, isFromOtherCluster, signatures) - f.DeleteChunks(ctx, entry.FullPath, chunksToDelete) + if !f.FilerConf.MatchStorageRule(string(entry.FullPath)).DisableChunkDeletion { + // a child's own path decides fsync, as when it is deleted alone + f.doDeleteChunks(ctx, chunksToDelete, false) + f.doDeleteChunks(ctx, fsyncChunksToDelete, true) + } return nil } diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index 0fe0f62d96..571e7b4256 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -4,6 +4,7 @@ import ( "container/heap" "context" "fmt" + "net/http" "strings" "sync" "time" @@ -362,7 +363,8 @@ func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []stri } // Delete files and classify outcomes - outcomes := deleteFilesAndClassify(ctx, f.volumeGrpcDialOption(), uniqueFileIdsSlice, lookupFunc) + fsyncIds := f.fsyncDeletionIds(uniqueFileIdsSlice) + outcomes := deleteFilesAndClassify(ctx, f.volumeGrpcDialOption(), uniqueFileIdsSlice, fsyncIds, lookupFunc) // Process outcomes var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int @@ -370,14 +372,15 @@ func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []stri for _, fileId := range uniqueFileIdsSlice { outcome := outcomes[fileId] + _, fsynced := fsyncIds[fileId] switch outcome.status { case deletionOutcomeSuccess: successCount++ - f.forgetDeletion(fileId) // confirmed gone: drop from durable ledger + f.forgetDeletion(fileId, fsynced) // confirmed gone: drop from durable ledger case deletionOutcomeNotFound: notFoundCount++ - f.forgetDeletion(fileId) // already gone: drop from durable ledger + f.forgetDeletion(fileId, fsynced) // already gone: drop from durable ledger case deletionOutcomeRetryable, deletionOutcomeNoResult: retryableErrorCount++ // Keep in the durable ledger: not confirmed, must survive restarts. @@ -432,10 +435,24 @@ type deletionOutcome struct { errorMsg string } -// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs -func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome { - // Perform deletion - results := operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, fileIds, lookupFunc) +// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs. +// Ids in fsyncIds are deleted durably. +func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, fsyncIds map[string]struct{}, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome { + var plainIds, syncedIds []string + for _, fileId := range fileIds { + if _, ok := fsyncIds[fileId]; ok { + syncedIds = append(syncedIds, fileId) + } else { + plainIds = append(plainIds, fileId) + } + } + var results []*volume_server_pb.DeleteResult + if len(plainIds) > 0 { + results = operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, plainIds, false, lookupFunc) + } + if len(syncedIds) > 0 { + results = append(results, operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, syncedIds, true, lookupFunc)...) + } // Group results by file ID to handle multiple results for replicated volumes resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult) @@ -446,7 +463,8 @@ func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, // Classify outcome for each file outcomes := make(map[string]deletionOutcome, len(fileIds)) for _, fileId := range fileIds { - outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId) + _, durable := fsyncIds[fileId] + outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId, durable) } return outcomes @@ -455,7 +473,9 @@ func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, // classifyDeletionOutcome examines all deletion results for a file ID and determines the overall outcome // Uses a single pass through results with early return for permanent errors (highest priority) // Priority: Permanent > Retryable > Success > Not Found -func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult) deletionOutcome { +// A durable delete the volume server could not sync is rolled back and +// answered with a 5xx, so for a durable delete that answer stays retryable. +func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult, durable bool) deletionOutcome { fileIdResults, found := resultsByFileId[fileId] if !found || len(fileIdResults) == 0 { return deletionOutcome{ @@ -476,7 +496,7 @@ func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume continue } - if isRetryableError(res.Error) { + if isRetryableError(res.Error) || (durable && res.Status >= http.StatusInternalServerError) { if firstRetryableError == "" { firstRetryableError = res.Error } @@ -553,7 +573,7 @@ func (f *Filer) loopProcessingDeletionRetry(lookupFunc func([]string) (map[strin // recorded. A surviving newer record still needs work, so // it goes back through the hot queue. if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) { - f.queueDeletions(item.FileId) + f.queueDeletions(false, item.FileId) } } if len(readyItems) == 0 { @@ -582,23 +602,25 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu } // Delete files and classify outcomes - outcomes := deleteFilesAndClassify(context.Background(), f.volumeGrpcDialOption(), fileIds, lookupFunc) + fsyncIds := f.fsyncDeletionIds(fileIds) + outcomes := deleteFilesAndClassify(context.Background(), f.volumeGrpcDialOption(), fileIds, fsyncIds, lookupFunc) // Process outcomes - iterate over readyItems to ensure all items are accounted for var successCount, notFoundCount, retryCount, permanentErrorCount int for _, item := range readyItems { outcome := outcomes[item.FileId] + _, fsynced := fsyncIds[item.FileId] switch outcome.status { case deletionOutcomeSuccess: successCount++ - f.DeletionRetryQueue.Remove(item) // Remove from queue (success) - f.forgetDeletion(item.FileId) // confirmed gone: drop from durable ledger + f.DeletionRetryQueue.Remove(item) // Remove from queue (success) + f.forgetDeletion(item.FileId, fsynced) // confirmed gone: drop from durable ledger glog.V(2).Infof("retry successful for %s after %d attempts", item.FileId, item.RetryCount) case deletionOutcomeNotFound: notFoundCount++ - f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted) - f.forgetDeletion(item.FileId) // already gone: drop from durable ledger + f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted) + f.forgetDeletion(item.FileId, fsynced) // already gone: drop from durable ledger case deletionOutcomeRetryable, deletionOutcomeNoResult: retryCount++ if outcome.status == deletionOutcomeNoResult { @@ -612,7 +634,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu // gave up: drop the record, unless the id was re-queued meanwhile // — a surviving newer record goes back through the hot queue. if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) { - f.queueDeletions(item.FileId) + f.queueDeletions(false, item.FileId) } glog.Warningf("permanent error on retry for %s after %d attempts: %s", item.FileId, item.RetryCount, outcome.errorMsg) } @@ -630,7 +652,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu } func (f *Filer) DeleteUncommittedChunks(ctx context.Context, chunks []*filer_pb.FileChunk) { - f.doDeleteChunks(ctx, chunks) + f.doDeleteChunks(ctx, chunks, false) } func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks []*filer_pb.FileChunk) { @@ -638,13 +660,13 @@ func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks if rule.DisableChunkDeletion { return } - f.doDeleteChunks(ctx, chunks) + f.doDeleteChunks(ctx, chunks, rule.GetFsync()) } -func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk) { +func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk, fsync bool) { for _, chunk := range chunks { if !chunk.IsChunkManifest { - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(fsync, chunk.GetFileIdString()) continue } dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk, f.MasterClient) @@ -652,15 +674,15 @@ func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk glog.V(0).InfofCtx(ctx, "failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) } for _, dChunk := range dataChunks { - f.queueDeletions(dChunk.GetFileIdString()) + f.queueDeletions(fsync, dChunk.GetFileIdString()) } - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(fsync, chunk.GetFileIdString()) } } func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) { for _, chunk := range chunks { - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(false, chunk.GetFileIdString()) } } diff --git a/weed/filer/filer_deletion_fsync_test.go b/weed/filer/filer_deletion_fsync_test.go new file mode 100644 index 0000000000..9b69eba869 --- /dev/null +++ b/weed/filer/filer_deletion_fsync_test.go @@ -0,0 +1,396 @@ +package filer + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +type fsyncRecordingVolumeServer struct { + volume_server_pb.UnimplementedVolumeServerServer + mu sync.Mutex + fsync map[string][]bool + failOnce map[string]string + failCode int32 + inFlight func(req *volume_server_pb.BatchDeleteRequest) +} + +func (v *fsyncRecordingVolumeServer) BatchDelete(_ context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) { + if v.inFlight != nil { + v.inFlight(req) + } + v.mu.Lock() + defer v.mu.Unlock() + resp := &volume_server_pb.BatchDeleteResponse{} + for _, fid := range req.FileIds { + v.fsync[fid] = append(v.fsync[fid], req.Fsync) + if msg, ok := v.failOnce[fid]; ok { + delete(v.failOnce, fid) + code := v.failCode + if code == 0 { + code = http.StatusServiceUnavailable + } + resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{FileId: fid, Status: code, Error: msg}) + continue + } + resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{FileId: fid, Status: http.StatusAccepted}) + } + return resp, nil +} + +func (v *fsyncRecordingVolumeServer) calls(fid string) []bool { + v.mu.Lock() + defer v.mu.Unlock() + return append([]bool(nil), v.fsync[fid]...) +} + +func startFsyncRecordingVolumeServer(t *testing.T) (*fsyncRecordingVolumeServer, func([]string) (map[string]*operation.LookupResult, error)) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + v := &fsyncRecordingVolumeServer{fsync: map[string][]bool{}, failOnce: map[string]string{}} + srv := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(srv, v) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + + port := lis.Addr().(*net.TCPAddr).Port + location := operation.Location{Url: "127.0.0.1:" + strconv.Itoa(port), GrpcPort: port} + lookup := func(vids []string) (map[string]*operation.LookupResult, error) { + m := make(map[string]*operation.LookupResult, len(vids)) + for _, vid := range vids { + m[vid] = &operation.LookupResult{VolumeOrFileId: vid, Locations: []operation.Location{location}} + } + return m, nil + } + return v, lookup +} + +func newFsyncRuleTestFiler(t *testing.T) *Filer { + t.Helper() + f := newLedgerTestFiler(newStubFilerStore()) + f.GrpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials()) + f.FilerConf = NewFilerConf() + if err := f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/", Fsync: proto.Bool(true)}); err != nil { + t.Fatal(err) + } + return f +} + +func TestDeleteChunksFsyncFollowsPathRule(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: "3,01637037d6"}}) + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: "3,02637037d6"}}) + f.DeleteUncommittedChunks(ctx, []*filer_pb.FileChunk{{FileId: "3,03637037d6"}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + + for fid, want := range map[string]bool{"3,01637037d6": true, "3,02637037d6": false, "3,03637037d6": false} { + got := v.calls(fid) + if len(got) != 1 || got[0] != want { + t.Errorf("%s: BatchDelete fsync calls %v, want [%v]", fid, got, want) + } + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } + if ids := f.fsyncDeletionIds([]string{"3,01637037d6"}); len(ids) != 0 { + t.Errorf("fsync mark kept after the delete was confirmed: %v", ids) + } +} + +func TestDeleteChunksFsyncSurvivesRetry(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + fid := "4,01637037d6" + v.failOnce[fid] = "volume 4 is read only" + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: fid}}) + f.queueDeletions(false, fid) + f.FlushFileIdDeletionQueue(ctx, lookup) + if f.DeletionRetryQueue.Size() != 1 { + t.Fatalf("retry queue size %d, want 1", f.DeletionRetryQueue.Size()) + } + + f.DeletionRetryQueue.lock.Lock() + f.DeletionRetryQueue.itemIndex[fid].NextRetryAt = time.Now().Add(-time.Second) + f.DeletionRetryQueue.lock.Unlock() + ready, _ := f.DeletionRetryQueue.GetReadyItems(DeletionRetryBatchSize) + f.processRetryBatch(ready, lookup) + + if got := v.calls(fid); len(got) != 2 || !got[0] || !got[1] { + t.Fatalf("BatchDelete fsync calls %v, want [true true]", got) + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } +} + +// readLedgerAsOlderFiler decodes the ledger the way a filer without fsync +// marks does: a JSON array, or a {parts, gen} manifest over its part keys. +func readLedgerAsOlderFiler(t *testing.T, f *Filer) []string { + t.Helper() + ctx := context.Background() + key := f.deletionLedgerKey() + raw, err := f.Store.KvGet(ctx, []byte(key)) + if err != nil { + t.Fatalf("read ledger: %v", err) + } + var ids []string + if json.Unmarshal(raw, &ids) == nil { + return ids + } + var manifest struct { + Parts int `json:"parts"` + Gen int `json:"gen,omitempty"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatalf("ledger %q is neither an array nor a manifest: %v", raw, err) + } + for i := 0; i < manifest.Parts; i++ { + part, err := f.Store.KvGet(ctx, deletionLedgerGenPartKey(key, manifest.Gen, i)) + if err != nil { + t.Fatalf("read part %d: %v", i, err) + } + var partIds []string + if err := json.Unmarshal(part, &partIds); err != nil { + t.Fatalf("part %d: %v", i, err) + } + ids = append(ids, partIds...) + } + return ids +} + +func TestDeletionLedgerKeepsFsyncMarks(t *testing.T) { + for _, n := range []int{1, 4000} { + t.Run(strconv.Itoa(n), func(t *testing.T) { + withPersistedConfig(t, true) + store := newStubFilerStore() + f := newLedgerTestFiler(store) + var plain, synced []string + for i := 0; i < n; i++ { + plain = append(plain, fmt.Sprintf("7,%08xbeefcafe", i)) + synced = append(synced, fmt.Sprintf("8,%08xbeefcafe", i)) + } + f.queueDeletions(false, plain...) + f.queueDeletions(true, synced...) + f.snapshotDeletionLedger() + + if got := readLedgerAsOlderFiler(t, f); len(got) != 2*n || !contains(got, plain[0]) || !contains(got, synced[n-1]) { + t.Fatalf("older filer reads %d ids, want all %d", len(got), 2*n) + } + + f2 := newLedgerTestFiler(store) + if !f2.reloadDeletionLedger() { + t.Fatalf("ledger should reload") + } + if got := f2.pendingDeletionCount(); got != 2*n { + t.Fatalf("recovered %d ids, want %d", got, 2*n) + } + if got := f2.fsyncDeletionIds(append(append([]string(nil), plain...), synced...)); len(got) != n { + t.Fatalf("recovered %d fsync marks, want %d", len(got), n) + } + if got := f2.fsyncDeletionIds(plain); len(got) != 0 { + t.Fatalf("plain ids came back marked: %d", len(got)) + } + + for _, id := range synced { + f2.forgetDeletion(id, true) + } + f2.snapshotDeletionLedger() + f3 := newLedgerTestFiler(store) + f3.reloadDeletionLedger() + if got := f3.fsyncDeletionIds(synced); len(got) != 0 || f3.pendingDeletionCount() != n { + t.Fatalf("after forgetting the marked ids: %d marks, %d pending", len(got), f3.pendingDeletionCount()) + } + if n == 1 { + if _, ok := readPersistedLedger(t, f2); !ok { + t.Fatalf("a ledger without marks should be a single array again") + } + } + }) + } +} + +func TestDeletionLedgerClaimKeepsFsyncMarks(t *testing.T) { + withPersistedConfig(t, true) + store := newStubFilerStore() + fOld := newLedgerTestFiler(store) + fOld.Dlm = lock_manager.NewDistributedLockManager("filer-old:8888") + fOld.queueDeletions(false, "1,01") + fOld.queueDeletions(true, "1,02") + fOld.snapshotDeletionLedger() + + fNew := newLedgerTestFiler(store) + fNew.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999") + if !fNew.reloadDeletionLedger() { + t.Fatalf("reload should claim the stranded ledger") + } + if got := fNew.fsyncDeletionIds([]string{"1,01", "1,02"}); len(got) != 1 { + t.Fatalf("claimed fsync marks %v, want only 1,02", got) + } + + fNext := newLedgerTestFiler(store) + fNext.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999") + fNext.reloadDeletionLedger() + if _, ok := fNext.fsyncDeletionIds([]string{"1,02"})["1,02"]; !ok { + t.Fatalf("claimed fsync mark was not persisted under the new key") + } +} + +func TestQueueDeletionsMarksBeforePublishing(t *testing.T) { + f := newLedgerTestFiler(newStubFilerStore()) + const n = 20000 + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < n; i++ { + f.queueDeletions(true, fmt.Sprintf("9,%08xbeefcafe", i)) + } + }() + seen := 0 + for seen < n { + f.FileIdDeletionQueue.Consume(func(ids []string) { + marked := f.fsyncDeletionIds(ids) + for _, id := range ids { + if _, ok := marked[id]; !ok { + t.Errorf("%s reached the deletion queue before its fsync mark", id) + } + } + seen += len(ids) + }) + if t.Failed() { + break + } + } + <-done +} + +func TestRecursiveDeleteFsyncFollowsChildPaths(t *testing.T) { + store := newStubFilerStore() + f := newTestFiler(t, store, NewFilerRemoteStorage()) + for prefix, fsync := range map[string]bool{"/data/": true, "/mix/important": true, "/keep": true, "/keep/tmp-": false} { + if err := f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: prefix, Fsync: proto.Bool(fsync)}); err != nil { + t.Fatal(err) + } + } + ctx := context.Background() + for _, dir := range []string{"/data", "/data/sub", "/mix", "/keep"} { + if err := f.CreateEntry(ctx, &Entry{FullPath: util.FullPath(dir), Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + "/data/a": "5,01637037d6", + "/data/sub/b": "5,02637037d6", + "/mix/important.txt": "5,03637037d6", + "/mix/other.txt": "5,04637037d6", + "/keep/y": "5,05637037d6", + "/keep/tmp-x": "5,06637037d6", + } + for path, fid := range files { + entry := &Entry{FullPath: util.FullPath(path), Attr: Attr{Mode: 0644, Mtime: time.Now()}, Chunks: []*filer_pb.FileChunk{{FileId: fid}}} + if err := f.CreateEntry(ctx, entry, nil, false, false, nil, true, 255); err != nil { + t.Fatal(err) + } + } + + for _, dir := range []string{"/data", "/mix", "/keep"} { + if err := f.DeleteEntryMetaAndData(ctx, util.FullPath(dir), true, false, true, false, nil, time.Time{}); err != nil { + t.Fatal(err) + } + } + + if n := f.pendingDeletionCount(); n != len(files) { + t.Fatalf("pending deletions %d, want %d", n, len(files)) + } + var fids []string + for _, fid := range files { + fids = append(fids, fid) + } + marked := f.fsyncDeletionIds(fids) + for path, fid := range files { + _, got := marked[fid] + if want := path != "/mix/other.txt" && path != "/keep/tmp-x"; got != want { + t.Errorf("%s: fsync mark %v, want %v", path, got, want) + } + } +} + +func TestPlainDeleteKeepsANewerFsyncMark(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + fid := "6,01637037d6" + var once sync.Once + v.inFlight = func(*volume_server_pb.BatchDeleteRequest) { + once.Do(func() { + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: fid}}) + }) + } + + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: fid}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + if _, ok := f.fsyncDeletionIds([]string{fid})[fid]; !ok || f.pendingDeletionCount() != 1 { + t.Fatalf("a plain delete dropped the fsync request queued while it ran") + } + + f.FlushFileIdDeletionQueue(ctx, lookup) + if got := v.calls(fid); len(got) != 2 || got[0] || !got[1] { + t.Fatalf("BatchDelete fsync calls %v, want [false true]", got) + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } +} + +func TestDurableDeleteRetriesAServerError(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + v.failCode = http.StatusInternalServerError + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + durable, plain := "7,01637037d6", "7,02637037d6" + v.failOnce[durable] = "sync /data/7.dat: input/output error" + v.failOnce[plain] = "sync /data/7.dat: input/output error" + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: durable}}) + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: plain}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + + f.DeletionRetryQueue.lock.Lock() + _, durableQueued := f.DeletionRetryQueue.itemIndex[durable] + _, plainQueued := f.DeletionRetryQueue.itemIndex[plain] + f.DeletionRetryQueue.lock.Unlock() + if !durableQueued || plainQueued { + t.Fatalf("retry queued: durable %v, plain %v; want durable only", durableQueued, plainQueued) + } + if _, ok := f.fsyncDeletionIds([]string{durable})[durable]; !ok || f.pendingDeletionCount() != 1 { + t.Fatalf("the failed durable delete lost its pending record or fsync mark") + } +} diff --git a/weed/filer/filer_deletion_persist.go b/weed/filer/filer_deletion_persist.go index 33b2371cf4..c1c18c0fd3 100644 --- a/weed/filer/filer_deletion_persist.go +++ b/weed/filer/filer_deletion_persist.go @@ -163,16 +163,16 @@ func (f *Filer) signalLedgerFlush() { // pipeline. It keeps the in-memory hot queue AND the durable ledger in sync. // Safe on a zero-value Filer (tests build struct literals without NewFiler): // the map is lazily created and the mutex is zero-value friendly. -func (f *Filer) queueDeletions(fileIds ...string) { +// fsync marks the ids for a durable volume delete; the mark stays until the +// record is forgotten, so a later enqueue without fsync keeps it. +func (f *Filer) queueDeletions(fsync bool, fileIds ...string) { if len(fileIds) == 0 { return } - // Hot path: existing in-memory queue, unchanged. - f.FileIdDeletionQueue.EnQueue(fileIds...) - // Durable ledger: record what still needs deleting. Every enqueue bumps the // id's epoch so an expiry-forget carrying an older epoch cannot erase the - // re-queued intent. + // re-queued intent. The record goes in before the id reaches the hot queue, + // so the delete loop never sees an id without its epoch and fsync mark. f.deletionLedgerLock.Lock() if f.pendingDeletions == nil { f.pendingDeletions = make(map[string]uint64, len(fileIds)) @@ -187,8 +187,18 @@ func (f *Filer) queueDeletions(fileIds ...string) { if !exists { f.deletionLedgerDirty = true } + if fsync { + if f.fsyncDeletions == nil { + f.fsyncDeletions = make(map[string]struct{}) + } + if _, marked := f.fsyncDeletions[id]; !marked { + f.fsyncDeletions[id] = struct{}{} + f.deletionLedgerDirty = true + } + } } f.deletionLedgerLock.Unlock() + f.FileIdDeletionQueue.EnQueue(fileIds...) f.signalLedgerFlush() } @@ -208,17 +218,41 @@ func (f *Filer) deletionEpoch(fileId string) uint64 { return f.pendingDeletions[fileId] } +// fsyncDeletionIds returns the subset of fileIds whose deletes must be durable. +func (f *Filer) fsyncDeletionIds(fileIds []string) map[string]struct{} { + f.deletionLedgerLock.Lock() + defer f.deletionLedgerLock.Unlock() + if len(f.fsyncDeletions) == 0 { + return nil + } + var ids map[string]struct{} + for _, id := range fileIds { + if _, ok := f.fsyncDeletions[id]; ok { + if ids == nil { + ids = make(map[string]struct{}) + } + ids[id] = struct{}{} + } + } + return ids +} + // forgetDeletion removes a fileId from the ledger once its deletion is terminal // (deleted or already absent: the chunks are gone, so any pending record — even // a re-queued one — refers to work that no longer exists). Non-terminal // outcomes must NOT call this so the entry survives until confirmed. -func (f *Filer) forgetDeletion(fileId string) { +// fsynced reports whether the finished delete was durable; if not, a record +// marked for fsync after that delete began stays for its queued copy, whose +// durable delete syncs the tombstone. +func (f *Filer) forgetDeletion(fileId string, fsynced bool) { if fileId == "" { return } f.deletionLedgerLock.Lock() - if _, exists := f.pendingDeletions[fileId]; exists { + _, marked := f.fsyncDeletions[fileId] + if _, exists := f.pendingDeletions[fileId]; exists && (fsynced || !marked) { delete(f.pendingDeletions, fileId) + delete(f.fsyncDeletions, fileId) f.deletionLedgerDirty = true } f.deletionLedgerLock.Unlock() @@ -237,6 +271,7 @@ func (f *Filer) forgetDeletionEpoch(fileId string, epoch uint64) bool { f.deletionLedgerLock.Lock() if cur, exists := f.pendingDeletions[fileId]; exists && cur == epoch { delete(f.pendingDeletions, fileId) + delete(f.fsyncDeletions, fileId) f.deletionLedgerDirty = true removed = true } @@ -280,10 +315,14 @@ func (f *Filer) snapshotDeletionLedger() { for id := range f.pendingDeletions { ids = append(ids, id) } + fsyncIds := make([]string, 0, len(f.fsyncDeletions)) + for id := range f.fsyncDeletions { + fsyncIds = append(fsyncIds, id) + } f.deletionLedgerDirty = false f.deletionLedgerLock.Unlock() - if err := f.writeDeletionLedger(ids); err != nil { + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { glog.Warningf("failed to persist deletion ledger (%d ids): %v", len(ids), err) f.deletionLedgerLock.Lock() f.deletionLedgerDirty = true @@ -306,7 +345,9 @@ func (f *Filer) tryUnblockDeletionLedger() bool { // atomically; a multipart set writes a new generation's part keys first and // only then the manifest, so a crash or failed write leaves the previously // published generation readable. Orphaned parts are retried on the next write. -func (f *Filer) writeDeletionLedger(ids []string) error { +// fsyncIds go into the trailing fsyncParts parts of a manifest, which older +// filers read as plain ids. +func (f *Filer) writeDeletionLedger(ids, fsyncIds []string) error { ctx := context.Background() key := f.deletionLedgerKey() @@ -332,7 +373,7 @@ func (f *Filer) writeDeletionLedger(ids []string) error { size = 2 return nil } - for _, id := range ids { + add := func(id string) error { if need := len(id) + 3; len(batch) > 0 && size+need > deletionLedgerPartSize { if err := flush(); err != nil { return err @@ -340,14 +381,39 @@ func (f *Filer) writeDeletionLedger(ids []string) error { } batch = append(batch, id) size += len(id) + 3 + return nil } - if len(batch) > 0 || len(parts) == 0 { + fsyncSet := make(map[string]struct{}, len(fsyncIds)) + for _, id := range fsyncIds { + fsyncSet[id] = struct{}{} + } + for _, id := range ids { + if _, ok := fsyncSet[id]; ok { + continue + } + if err := add(id); err != nil { + return err + } + } + if len(batch) > 0 || (len(parts) == 0 && len(fsyncSet) == 0) { if err := flush(); err != nil { return err } } + plainParts := len(parts) + for id := range fsyncSet { + if err := add(id); err != nil { + return err + } + } + if len(batch) > 0 { + if err := flush(); err != nil { + return err + } + } + fsyncParts := len(parts) - plainParts - if len(parts) == 1 { + if len(parts) == 1 && fsyncParts == 0 { if err := f.Store.KvPut(ctx, []byte(key), parts[0]); err != nil { return err } @@ -365,10 +431,7 @@ func (f *Filer) writeDeletionLedger(ids []string) error { } wrote++ } - manifest, _ := json.Marshal(struct { - Parts int `json:"parts"` - Gen int `json:"gen,omitempty"` - }{len(parts), gen}) + manifest, _ := json.Marshal(deletionLedgerManifest{Parts: len(parts), Gen: gen, FsyncParts: fsyncParts}) if err := f.Store.KvPut(ctx, []byte(key), manifest); err != nil { f.markStaleLedgerParts(key, gen, wrote) return err @@ -523,42 +586,49 @@ func (f *Filer) pruneLedgerIndex(ctx context.Context, key string) { _ = f.Store.KvPut(ctx, []byte(KvKeyDeletionLedgerIndex), payload) } +type deletionLedgerManifest struct { + Parts int `json:"parts"` + Gen int `json:"gen,omitempty"` + FsyncParts int `json:"fsyncParts,omitempty"` +} + // readDeletionLedger reads the manifest key: a JSON array is the whole set; a // {"parts":N,"gen":G} manifest points at that generation's part keys. A part // the manifest references but the store lacks is corruption, not absence, so -// it surfaces as a wrapped error rather than ErrKvNotFound. -func (f *Filer) readDeletionLedger(key string) (ids []string, gen, parts int, err error) { +// it surfaces as a wrapped error rather than ErrKvNotFound. ids holds every +// id; fsyncIds repeats the ones from the trailing fsyncParts parts. +func (f *Filer) readDeletionLedger(key string) (ids, fsyncIds []string, gen, parts int, err error) { ctx := context.Background() payload, err := f.Store.KvGet(ctx, []byte(key)) if err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } if !bytes.HasPrefix(bytes.TrimSpace(payload), []byte("{")) { if err := json.Unmarshal(payload, &ids); err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } - return ids, 0, 0, nil - } - var manifest struct { - Parts int `json:"parts"` - Gen int `json:"gen,omitempty"` + return ids, nil, 0, 0, nil } + var manifest deletionLedgerManifest if err := json.Unmarshal(payload, &manifest); err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } for i := 0; i < manifest.Parts; i++ { partPayload, err := f.Store.KvGet(ctx, deletionLedgerGenPartKey(key, manifest.Gen, i)) if err != nil { - return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s unreadable: %w", i, key, err) + return nil, nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s unreadable: %w", i, key, err) } var part []string if err := json.Unmarshal(partPayload, &part); err != nil { - return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s corrupt: %w", i, key, err) + return nil, nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s corrupt: %w", i, key, err) } ids = append(ids, part...) + if i >= manifest.Parts-manifest.FsyncParts { + fsyncIds = append(fsyncIds, part...) + } } - return ids, manifest.Gen, manifest.Parts, nil + return ids, fsyncIds, manifest.Gen, manifest.Parts, nil } // recoverForeignDeletionLedgers runs when the scoped key is absent: the ledger @@ -568,18 +638,19 @@ func (f *Filer) readDeletionLedger(key string) (ids []string, gen, parts int, er // deleted only after that write succeeds. A live peer's ledger claimed here is // rewritten by the peer's next snapshot, so the ids only get processed twice — // deletions are not owner-specific. -func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err error) { +func (f *Filer) recoverForeignDeletionLedgers(key string) (ids, fsyncIds []string, err error) { ctx := context.Background() claimed := map[string]ledgerClaim{} // source key -> what was read - lIds, lGen, lParts, lErr := f.readDeletionLedger(KvKeyDeletionLedger) + lIds, lFsyncIds, lGen, lParts, lErr := f.readDeletionLedger(KvKeyDeletionLedger) switch { case lErr == nil: ids = append(ids, lIds...) + fsyncIds = append(fsyncIds, lFsyncIds...) claimed[KvKeyDeletionLedger] = ledgerClaim{lIds, lGen, lParts} f.loadStaleLedgerParts(ctx, KvKeyDeletionLedger) case lErr != ErrKvNotFound: - return nil, lErr + return nil, nil, lErr } // Any read failure aborts the whole claim so reload retries: skipping an @@ -587,36 +658,37 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err // the index is never searched again. indexKeys, idxErr := f.readLedgerIndex(ctx) if idxErr != nil && idxErr != ErrKvNotFound { - return nil, idxErr + return nil, nil, idxErr } for _, other := range indexKeys { if other == key { continue } - oIds, oGen, oParts, oErr := f.readDeletionLedger(other) + oIds, oFsyncIds, oGen, oParts, oErr := f.readDeletionLedger(other) if oErr == ErrKvNotFound { f.pruneLedgerIndex(ctx, other) continue } if oErr != nil { - return nil, fmt.Errorf("deletion ledger %s unreadable: %w", other, oErr) + return nil, nil, fmt.Errorf("deletion ledger %s unreadable: %w", other, oErr) } ids = append(ids, oIds...) + fsyncIds = append(fsyncIds, oFsyncIds...) claimed[other] = ledgerClaim{oIds, oGen, oParts} f.loadStaleLedgerParts(ctx, other) } if len(claimed) == 0 { - return nil, ErrKvNotFound + return nil, nil, ErrKvNotFound } - if err := f.writeDeletionLedger(ids); err != nil { - return nil, err + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { + return nil, nil, err } mergedMore := false for src, claim := range claimed { // A source republished since we read it belongs to a live filer // (or a racing claimer): merge the newer ids and leave it in place. - curIds, curGen, curParts, rerr := f.readDeletionLedger(src) + curIds, curFsyncIds, curGen, curParts, rerr := f.readDeletionLedger(src) if rerr != nil || curGen != claim.gen || curParts != claim.parts || !sameStringSet(curIds, claim.ids) { if rerr == nil { for _, id := range curIds { @@ -625,6 +697,12 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err mergedMore = true } } + for _, id := range curFsyncIds { + if !containsId(fsyncIds, id) { + fsyncIds = append(fsyncIds, id) + mergedMore = true + } + } } glog.V(0).Infof("deletion ledger %s changed while claiming; leaving it for its owner", src) continue @@ -637,11 +715,11 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err if mergedMore { // Ids from a changed source are durable only under that source's key; // rewrite our ledger so they survive under ours too. - if err := f.writeDeletionLedger(ids); err != nil { - return nil, err + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { + return nil, nil, err } } - return ids, nil + return ids, fsyncIds, nil } func containsId(ids []string, id string) bool { @@ -692,10 +770,10 @@ func (f *Filer) reloadDeletionLedger() bool { } key := f.deletionLedgerKey() - ids, gen, parts, err := f.readDeletionLedger(key) + ids, fsyncIds, gen, parts, err := f.readDeletionLedger(key) stateWritten := false if err == ErrKvNotFound && key != KvKeyDeletionLedger { - ids, err = f.recoverForeignDeletionLedgers(key) + ids, fsyncIds, err = f.recoverForeignDeletionLedgers(key) stateWritten = err == nil } if err != nil { @@ -725,6 +803,14 @@ func (f *Filer) reloadDeletionLedger() bool { } } } + for _, id := range fsyncIds { + if _, exists := f.pendingDeletions[id]; exists { + if f.fsyncDeletions == nil { + f.fsyncDeletions = make(map[string]struct{}, len(fsyncIds)) + } + f.fsyncDeletions[id] = struct{}{} + } + } } if !stateWritten { f.deletionLedgerGen = gen @@ -751,7 +837,7 @@ func (f *Filer) reloadDeletionLedger() bool { // for peer meta-aggregation to settle. The ledger is deliberately NOT // cleared here: it shrinks only as the delete pipeline confirms each // id terminal, so a second crash mid-recovery replays everything. - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) glog.V(0).Infof("re-queued %d recovered pending deletions", len(ids)) }() return true diff --git a/weed/filer/filer_deletion_persist_test.go b/weed/filer/filer_deletion_persist_test.go index d9f82f54b7..3432a37921 100644 --- a/weed/filer/filer_deletion_persist_test.go +++ b/weed/filer/filer_deletion_persist_test.go @@ -64,7 +64,7 @@ func TestDeletionLedgerSnapshotAndRecover(t *testing.T) { store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("1,01", "1,02", "1,03") + f.queueDeletions(false, "1,01", "1,02", "1,03") f.snapshotDeletionLedger() persisted, ok := readPersistedLedger(t, f) @@ -81,7 +81,7 @@ func TestDeletionLedgerSnapshotAndRecover(t *testing.T) { } // Confirmed-gone for one id, then snapshot again. - f.forgetDeletion("1,02") + f.forgetDeletion("1,02", false) f.snapshotDeletionLedger() persisted, _ = readPersistedLedger(t, f) if len(persisted) != 2 { @@ -130,7 +130,7 @@ func TestDeletionLedgerRetryKeepsEntry(t *testing.T) { store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("9,01") + f.queueDeletions(false, "9,01") f.snapshotDeletionLedger() // A retryable outcome deliberately does NOT call forgetDeletion, so the id @@ -152,7 +152,7 @@ func TestDeletionLedgerDisabled(t *testing.T) { withPersistedConfig(t, true) store := newStubFilerStore() fOn := newLedgerTestFiler(store) - fOn.queueDeletions("5,01") + fOn.queueDeletions(false, "5,01") fOn.snapshotDeletionLedger() if _, ok := readPersistedLedger(t, fOn); !ok { t.Fatalf("setup: expected a persisted ledger with persist enabled") @@ -163,7 +163,7 @@ func TestDeletionLedgerDisabled(t *testing.T) { // 1. Snapshot does not touch KV (no new writes, no clear). f := newLedgerTestFiler(store) - f.queueDeletions("6,01") // in-memory only + f.queueDeletions(false, "6,01") // in-memory only f.snapshotDeletionLedger() persisted, ok := readPersistedLedger(t, f) if !ok { @@ -192,8 +192,8 @@ func TestDeletionLedgerScopedPerFiler(t *testing.T) { fB := newLedgerTestFiler(store) fB.Dlm = lock_manager.NewDistributedLockManager("filer-b:8888") - fA.queueDeletions("1,01") - fB.queueDeletions("2,02") + fA.queueDeletions(false, "1,01") + fB.queueDeletions(false, "2,02") fA.snapshotDeletionLedger() fB.snapshotDeletionLedger() @@ -231,7 +231,7 @@ func TestDeletionLedgerChunked(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() raw, err := f.Store.KvGet(context.Background(), []byte(f.deletionLedgerKey())) @@ -250,7 +250,7 @@ func TestDeletionLedgerChunked(t *testing.T) { // Forget everything; the next snapshot is a single value again and the // part keys are removed. for _, id := range ids { - f2.forgetDeletion(id) + f2.forgetDeletion(id, false) } f2.snapshotDeletionLedger() raw, err = f.Store.KvGet(context.Background(), []byte(f2.deletionLedgerKey())) @@ -274,7 +274,7 @@ func TestDeletionLedgerReadFailureBlocksPersistence(t *testing.T) { t.Fatalf("reload should report the ledger as unusable") } - f.queueDeletions("1,01") + f.queueDeletions(false, "1,01") f.snapshotDeletionLedger() if len(store.kv) != 0 { t.Fatalf("snapshot must not overwrite a ledger that was never read, wrote %v", store.kv) @@ -297,7 +297,7 @@ func TestDeletionLedgerRecoveryKeepsIdsPending(t *testing.T) { withPersistedConfig(t, true) store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("1,01", "1,02") + f.queueDeletions(false, "1,01", "1,02") f.snapshotDeletionLedger() util.GetViper().Set("filer.deleteQueue.recoveryGrace", time.Hour) @@ -326,11 +326,11 @@ func TestDeletionLedgerZeroValueFiler(t *testing.T) { DeletionRetryQueue: NewDeletionRetryQueue(), deletionQuit: make(chan struct{}), } - f.queueDeletions("0,01") + f.queueDeletions(false, "0,01") if f.pendingDeletionCount() != 1 { t.Fatalf("zero-value filer should lazily init the ledger, got %d", f.pendingDeletionCount()) } - f.forgetDeletion("0,01") + f.forgetDeletion("0,01", false) if f.pendingDeletionCount() != 0 { t.Fatalf("forget on zero-value filer failed, got %d", f.pendingDeletionCount()) } @@ -352,7 +352,7 @@ func TestDeletionLedgerMissingPartBlocks(t *testing.T) { if f.reloadDeletionLedger() { t.Fatalf("a manifest referencing a missing part is corrupt, not absent") } - f.queueDeletions("9,99") + f.queueDeletions(false, "9,99") f.snapshotDeletionLedger() if got := string(store.kv[key]); got != `{"parts":2,"gen":4}` { t.Fatalf("unread ledger must not be overwritten, got %q", got) @@ -367,7 +367,7 @@ func TestDeletionLedgerAddressChangeClaim(t *testing.T) { store := newStubFilerStore() fOld := newLedgerTestFiler(store) fOld.Dlm = lock_manager.NewDistributedLockManager("filer-old:8888") - fOld.queueDeletions("1,01", "1,02") + fOld.queueDeletions(false, "1,01", "1,02") fOld.snapshotDeletionLedger() fNew := newLedgerTestFiler(store) @@ -403,14 +403,14 @@ func TestDeletionLedgerGenerationPublish(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() key := f.deletionLedgerKey() if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 1, 0)); err != nil { t.Fatalf("generation 1 part must exist, got %v", err) } - f.queueDeletions("8,08") + f.queueDeletions(false, "8,08") f.snapshotDeletionLedger() if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 2, 0)); err != nil { t.Fatalf("generation 2 part must exist, got %v", err) @@ -439,18 +439,18 @@ func TestDeletionLedgerStalePartsRetried(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() key := f.deletionLedgerKey() store.kvDeleteErr = errors.New("delete down") - f.queueDeletions("8,08") + f.queueDeletions(false, "8,08") f.snapshotDeletionLedger() if len(f.deletionLedgerStale) == 0 { t.Fatalf("failed part deletes must be tracked for retry") } store.kvDeleteErr = nil - f.queueDeletions("8,09") + f.queueDeletions(false, "8,09") f.snapshotDeletionLedger() if len(f.deletionLedgerStale) != 0 { t.Fatalf("stale parts must flush once deletes work, got %v", f.deletionLedgerStale) @@ -556,9 +556,9 @@ func TestForgetDeletionEpochSkipsNewer(t *testing.T) { withPersistedConfig(t, true) f := newLedgerTestFiler(newStubFilerStore()) - f.queueDeletions("1,01") + f.queueDeletions(false, "1,01") oldEpoch := f.deletionEpoch("1,01") - f.queueDeletions("1,01") // re-enqueue bumps the epoch + f.queueDeletions(false, "1,01") // re-enqueue bumps the epoch newEpoch := f.deletionEpoch("1,01") if oldEpoch == newEpoch { t.Fatalf("re-enqueue must bump the epoch") diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index 297f665150..d7b9974051 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -45,11 +45,12 @@ func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc. return } - return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, lookupFunc) + return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, false, lookupFunc) } -func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult { +// fsync asks each volume server to make the deletes durable before answering. +func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, fsync bool, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult { var ret []*volume_server_pb.DeleteResult @@ -117,7 +118,7 @@ func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.Di go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false) + resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false, fsync) }(server, fidList) } @@ -133,7 +134,7 @@ func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.Di // DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc // Returns individual results for each file ID. Check result.Error for per-file failures. -func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult { +func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool, fsync bool) []*volume_server_pb.DeleteResult { var ret []*volume_server_pb.DeleteResult @@ -142,6 +143,7 @@ func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerA req := &volume_server_pb.BatchDeleteRequest{ FileIds: fileIds, SkipCookieCheck: !includeCookie, + Fsync: fsync, } resp, err := volumeServerClient.BatchDelete(ctx, req) diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 905813e43b..e56b324bc8 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -147,6 +147,7 @@ service VolumeServer { message BatchDeleteRequest { repeated string file_ids = 1; bool skip_cookie_check = 2; + bool fsync = 3; } message BatchDeleteResponse { diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 4e131ce276..d3d485f009 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -185,6 +185,7 @@ type BatchDeleteRequest struct { state protoimpl.MessageState `protogen:"open.v1"` FileIds []string `protobuf:"bytes,1,rep,name=file_ids,json=fileIds,proto3" json:"file_ids,omitempty"` SkipCookieCheck bool `protobuf:"varint,2,opt,name=skip_cookie_check,json=skipCookieCheck,proto3" json:"skip_cookie_check,omitempty"` + Fsync bool `protobuf:"varint,3,opt,name=fsync,proto3" json:"fsync,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -233,6 +234,13 @@ func (x *BatchDeleteRequest) GetSkipCookieCheck() bool { return false } +func (x *BatchDeleteRequest) GetFsync() bool { + if x != nil { + return x.Fsync + } + return false +} + type BatchDeleteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Results []*DeleteResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` @@ -7334,10 +7342,11 @@ const file_volume_server_proto_rawDesc = "" + "\x13volume_server.proto\x12\x10volume_server_pb\x1a\fremote.proto\"O\n" + "\x11VolumeServerState\x12 \n" + "\vmaintenance\x18\x01 \x01(\bR\vmaintenance\x12\x18\n" + - "\aversion\x18\x02 \x01(\rR\aversion\"[\n" + + "\aversion\x18\x02 \x01(\rR\aversion\"q\n" + "\x12BatchDeleteRequest\x12\x19\n" + "\bfile_ids\x18\x01 \x03(\tR\afileIds\x12*\n" + - "\x11skip_cookie_check\x18\x02 \x01(\bR\x0fskipCookieCheck\"O\n" + + "\x11skip_cookie_check\x18\x02 \x01(\bR\x0fskipCookieCheck\x12\x14\n" + + "\x05fsync\x18\x03 \x01(\bR\x05fsync\"O\n" + "\x13BatchDeleteResponse\x128\n" + "\aresults\x18\x01 \x03(\v2\x1e.volume_server_pb.DeleteResultR\aresults\"\x83\x01\n" + "\fDeleteResult\x12\x17\n" + diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index f15bdc8dc1..4092e5a0e8 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -3168,7 +3168,7 @@ func (s3a *S3ApiServer) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) { } // Attempt deletion using the operation package's batch delete with custom lookup - deleteResults := operation.DeleteFileIdsWithLookupVolumeId(context.Background(), s3a.option.GrpcDialOption, fileIds, lookupFunc) + deleteResults := operation.DeleteFileIdsWithLookupVolumeId(context.Background(), s3a.option.GrpcDialOption, fileIds, false, lookupFunc) // Log results - track successes and failures successCount := 0 diff --git a/weed/server/volume_grpc_batch_delete.go b/weed/server/volume_grpc_batch_delete.go index df996b38ab..c08abb5fc6 100644 --- a/weed/server/volume_grpc_batch_delete.go +++ b/weed/server/volume_grpc_batch_delete.go @@ -105,7 +105,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B // the delete cancels in-flight writes carrying this cookie n.Cookie = fidCookie } - if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, false); err != nil { + if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, req.Fsync); err != nil { resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{ FileId: fid, Status: http.StatusInternalServerError, diff --git a/weed/server/volume_grpc_batch_delete_test.go b/weed/server/volume_grpc_batch_delete_test.go index 927b101f61..4f236245c2 100644 --- a/weed/server/volume_grpc_batch_delete_test.go +++ b/weed/server/volume_grpc_batch_delete_test.go @@ -2,15 +2,25 @@ package weed_server import ( "context" + "fmt" "net" + "net/http" + "sync/atomic" "testing" + "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" ) // TestBatchDelete_DeniesNonWhitelistedPeer verifies that BatchDelete refuses @@ -42,3 +52,46 @@ func TestBatchDelete_DeniesNonWhitelistedPeer(t *testing.T) { t.Fatalf("expected no per-file results on auth failure, got %d", len(resp.Results)) } } + +type syncCountingBackend struct { + backend.BackendStorageFile + syncs atomic.Int32 +} + +func (b *syncCountingBackend) Sync() error { + b.syncs.Add(1) + return b.BackendStorageFile.Sync() +} + +func TestBatchDeleteFsyncsOnlyWhenAsked(t *testing.T) { + for _, fsync := range []bool{false, true} { + t.Run(fmt.Sprint(fsync), func(t *testing.T) { + store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{t.TempDir()}, []int32{10}, []util.MinFreeSpace{{}}, "", storage.NeedleMapInMemory, + []types.DiskType{types.HardDriveType}, nil, 0, stats.DefaultDiskIOProbeConfig()) + t.Cleanup(store.Close) + require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0)) + n := &needle.Needle{Id: 1, Cookie: 0x1234, Data: []byte("chunk")} + n.Checksum = needle.NewCRC(n.Data) + _, err := store.WriteVolumeNeedle(1, n, true, false) + require.NoError(t, err) + + v := store.GetVolume(1) + counting := &syncCountingBackend{BackendStorageFile: v.DataBackend} + v.DataBackend = counting + + resp, err := (&VolumeServer{store: store}).BatchDelete(context.Background(), &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{needle.NewFileIdFromNeedle(1, n).String()}, + Fsync: fsync, + }) + require.NoError(t, err) + require.Len(t, resp.Results, 1) + require.Equal(t, int32(http.StatusAccepted), resp.Results[0].Status, resp.Results[0].Error) + if fsync { + require.Positive(t, counting.syncs.Load(), "the tombstone was not synced") + } else { + require.Zero(t, counting.syncs.Load(), "a delete that did not ask for fsync paid for one") + } + }) + } +} diff --git a/weed/shell/command_fs_merge_volumes.go b/weed/shell/command_fs_merge_volumes.go index 5bf8c49d26..65f928f897 100644 --- a/weed/shell/command_fs_merge_volumes.go +++ b/weed/shell/command_fs_merge_volumes.go @@ -493,7 +493,7 @@ func deleteOrphanedNeedles(commandEnv *CommandEnv, entryPath util.FullPath, need continue } for _, loc := range locations { - results := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie) + results := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie, false) // Summarize per server: an unreachable volume server returns one // error per needle, which for manifest-heavy files can mean // hundreds of near-identical lines. Keep the first error as the diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index edd9416446..5c47cf86bd 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -769,7 +769,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me deleteResults := operation.DeleteFileIdsAtOneVolumeServer( context.Background(), pb.NewServerAddressFromDataNode(target.location.dataNode), - vcd.grpcDialOption(), fidList, false) + vcd.grpcDialOption(), fidList, false, false) // Check for errors in results for _, deleteResult := range deleteResults { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index a7d7ec918b..276130a2e5 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -1142,7 +1142,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - deleteResults := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), server, c.env.option.GrpcDialOption, fidList, false) + deleteResults := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), server, c.env.option.GrpcDialOption, fidList, false, false) if deleteResults != nil { resultChan <- deleteResults }