diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto index b36932c9eb..2faea41a85 100644 --- a/seaweed-volume/proto/volume_server.proto +++ b/seaweed-volume/proto/volume_server.proto @@ -147,6 +147,7 @@ service VolumeServer { message BatchDeleteRequest { repeated string file_ids = 1; bool skip_cookie_check = 2; + bool fsync = 3; } message BatchDeleteResponse { diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 23eb10d2db..9d263dce40 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -1276,7 +1276,7 @@ impl VolumeServer for VolumeGrpcService { Some(vol) if vol.is_no_write_or_delete() => { Err(crate::storage::volume::VolumeError::ReadOnly(file_id.volume_id)) } - Some(vol) => vol.delete_needle(&mut n, false), + Some(vol) => vol.delete_needle(&mut n, req.fsync), }; match delete_result { Ok(size) => { @@ -14237,6 +14237,7 @@ mod tests { .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec![fid.to_string()], skip_cookie_check, + fsync: false, })) .await .unwrap() @@ -14290,6 +14291,57 @@ mod tests { ); } + /// Go's TestBatchDeleteFsyncsOnlyWhenAsked. + #[tokio::test] + async fn test_batch_delete_fsyncs_only_when_asked() { + let (service, _tmp) = make_local_service_with_volume("", None); + for (id, fsync) in [(0x6e7a_0c01u64, false), (0x6e7a_0c02, true)] { + let mut n = Needle { + id: NeedleId(id), + cookie: Cookie(0x3344), + data: b"chunk".to_vec(), + data_size: 5, + ..Needle::default() + }; + service + .state + .store + .write() + .unwrap() + .write_volume_needle(VolumeId(1), &mut n, false) + .unwrap(); + let syncs = || { + let store = service.state.store.read().unwrap(); + let (_, vol) = store.find_volume(VolumeId(1)).unwrap(); + vol.sync_counts_for_test() + }; + let before = syncs(); + let fid = needle::FileId::new(VolumeId(1), NeedleId(id), Cookie(0x3344)); + let resp = service + .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { + file_ids: vec![fid.to_string()], + skip_cookie_check: true, + fsync, + })) + .await + .unwrap() + .into_inner(); + assert_eq!(resp.results[0].status, 202, "{:?}", resp.results); + let after = syncs(); + if fsync { + assert!( + after.0 > before.0 && after.1 > before.1, + "the tombstone was not synced: {before:?} -> {after:?}" + ); + } else { + assert_eq!( + after, before, + "a delete that did not ask for fsync paid for one" + ); + } + } + } + const GO_EC_NEEDLE_NOT_FOUND: &str = "locate in local ec volume: FindNeedleFromEcx: needle not found"; @@ -16327,6 +16379,7 @@ mod tests { Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec!["1,b00003344".to_string()], skip_cookie_check: false, + fsync: false, }) }; @@ -16376,6 +16429,7 @@ mod tests { .batch_delete(Request::new(volume_server_pb::BatchDeleteRequest { file_ids: vec!["2,b00003344".to_string()], skip_cookie_check, + fsync: false, })) .await .unwrap() diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 73af9c0058..53b5ad9194 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -683,6 +683,7 @@ async fn batch_delete_file_ids( .batch_delete(volume_server_pb::BatchDeleteRequest { file_ids: batch.clone(), skip_cookie_check: true, + fsync: false, }) .await .map_err(|e| format!("batch delete {}: {}", grpc_addr, e))? diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 1b9b10664f..48c8dd7631 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -107,9 +107,11 @@ type Filer struct { // to the store so a restart does not leak chunks. Guarded by // deletionLedgerLock; nil-safe for Filer literals in tests. // pendingDeletions maps each id to its enqueue epoch so an expiry-forget - // cannot erase a re-queued id. + // cannot erase a re-queued id. fsyncDeletions marks pending ids whose + // volume deletes must be durable. deletionLedgerLock sync.Mutex pendingDeletions map[string]uint64 + fsyncDeletions map[string]struct{} deletionSeq uint64 deletionLedgerDirty bool deletionLedgerParts int diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 2d82d532f0..e8a0e076cb 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -205,7 +205,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry } //collect all the chunks of this layer and delete them together at the end - var chunksToDelete []*filer_pb.FileChunk + var chunksToDelete, fsyncChunksToDelete []*filer_pb.FileChunk lastFileName := "" includeLastFile := false listedChildren := !isDeletingBucket || !f.Store.CanDropWholeBucket() @@ -246,15 +246,19 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry break } f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, sub), nil, shouldDeleteChunks, isFromOtherCluster, nil) + subChunksToDelete := &chunksToDelete + if shouldDeleteChunks && f.FilerConf.MatchStorageRule(string(sub.FullPath)).GetFsync() { + subChunksToDelete = &fsyncChunksToDelete + } if len(sub.HardLinkId) != 0 { // hard link chunk data are deleted separately err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId}) if shouldDeleteChunks && f.hardLinkIsOrphaned(ctx, sub.HardLinkId) { - chunksToDelete = append(chunksToDelete, sub.GetChunks()...) + *subChunksToDelete = append(*subChunksToDelete, sub.GetChunks()...) } } else { if shouldDeleteChunks { - chunksToDelete = append(chunksToDelete, sub.GetChunks()...) + *subChunksToDelete = append(*subChunksToDelete, sub.GetChunks()...) } } } @@ -280,7 +284,11 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry } f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, entry), nil, shouldDeleteChunks, isFromOtherCluster, signatures) - f.DeleteChunks(ctx, entry.FullPath, chunksToDelete) + if !f.FilerConf.MatchStorageRule(string(entry.FullPath)).DisableChunkDeletion { + // a child's own path decides fsync, as when it is deleted alone + f.doDeleteChunks(ctx, chunksToDelete, false) + f.doDeleteChunks(ctx, fsyncChunksToDelete, true) + } return nil } diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index 0fe0f62d96..571e7b4256 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -4,6 +4,7 @@ import ( "container/heap" "context" "fmt" + "net/http" "strings" "sync" "time" @@ -362,7 +363,8 @@ func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []stri } // Delete files and classify outcomes - outcomes := deleteFilesAndClassify(ctx, f.volumeGrpcDialOption(), uniqueFileIdsSlice, lookupFunc) + fsyncIds := f.fsyncDeletionIds(uniqueFileIdsSlice) + outcomes := deleteFilesAndClassify(ctx, f.volumeGrpcDialOption(), uniqueFileIdsSlice, fsyncIds, lookupFunc) // Process outcomes var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int @@ -370,14 +372,15 @@ func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []stri for _, fileId := range uniqueFileIdsSlice { outcome := outcomes[fileId] + _, fsynced := fsyncIds[fileId] switch outcome.status { case deletionOutcomeSuccess: successCount++ - f.forgetDeletion(fileId) // confirmed gone: drop from durable ledger + f.forgetDeletion(fileId, fsynced) // confirmed gone: drop from durable ledger case deletionOutcomeNotFound: notFoundCount++ - f.forgetDeletion(fileId) // already gone: drop from durable ledger + f.forgetDeletion(fileId, fsynced) // already gone: drop from durable ledger case deletionOutcomeRetryable, deletionOutcomeNoResult: retryableErrorCount++ // Keep in the durable ledger: not confirmed, must survive restarts. @@ -432,10 +435,24 @@ type deletionOutcome struct { errorMsg string } -// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs -func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome { - // Perform deletion - results := operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, fileIds, lookupFunc) +// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs. +// Ids in fsyncIds are deleted durably. +func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, fsyncIds map[string]struct{}, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome { + var plainIds, syncedIds []string + for _, fileId := range fileIds { + if _, ok := fsyncIds[fileId]; ok { + syncedIds = append(syncedIds, fileId) + } else { + plainIds = append(plainIds, fileId) + } + } + var results []*volume_server_pb.DeleteResult + if len(plainIds) > 0 { + results = operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, plainIds, false, lookupFunc) + } + if len(syncedIds) > 0 { + results = append(results, operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, syncedIds, true, lookupFunc)...) + } // Group results by file ID to handle multiple results for replicated volumes resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult) @@ -446,7 +463,8 @@ func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, // Classify outcome for each file outcomes := make(map[string]deletionOutcome, len(fileIds)) for _, fileId := range fileIds { - outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId) + _, durable := fsyncIds[fileId] + outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId, durable) } return outcomes @@ -455,7 +473,9 @@ func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, // classifyDeletionOutcome examines all deletion results for a file ID and determines the overall outcome // Uses a single pass through results with early return for permanent errors (highest priority) // Priority: Permanent > Retryable > Success > Not Found -func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult) deletionOutcome { +// A durable delete the volume server could not sync is rolled back and +// answered with a 5xx, so for a durable delete that answer stays retryable. +func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult, durable bool) deletionOutcome { fileIdResults, found := resultsByFileId[fileId] if !found || len(fileIdResults) == 0 { return deletionOutcome{ @@ -476,7 +496,7 @@ func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume continue } - if isRetryableError(res.Error) { + if isRetryableError(res.Error) || (durable && res.Status >= http.StatusInternalServerError) { if firstRetryableError == "" { firstRetryableError = res.Error } @@ -553,7 +573,7 @@ func (f *Filer) loopProcessingDeletionRetry(lookupFunc func([]string) (map[strin // recorded. A surviving newer record still needs work, so // it goes back through the hot queue. if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) { - f.queueDeletions(item.FileId) + f.queueDeletions(false, item.FileId) } } if len(readyItems) == 0 { @@ -582,23 +602,25 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu } // Delete files and classify outcomes - outcomes := deleteFilesAndClassify(context.Background(), f.volumeGrpcDialOption(), fileIds, lookupFunc) + fsyncIds := f.fsyncDeletionIds(fileIds) + outcomes := deleteFilesAndClassify(context.Background(), f.volumeGrpcDialOption(), fileIds, fsyncIds, lookupFunc) // Process outcomes - iterate over readyItems to ensure all items are accounted for var successCount, notFoundCount, retryCount, permanentErrorCount int for _, item := range readyItems { outcome := outcomes[item.FileId] + _, fsynced := fsyncIds[item.FileId] switch outcome.status { case deletionOutcomeSuccess: successCount++ - f.DeletionRetryQueue.Remove(item) // Remove from queue (success) - f.forgetDeletion(item.FileId) // confirmed gone: drop from durable ledger + f.DeletionRetryQueue.Remove(item) // Remove from queue (success) + f.forgetDeletion(item.FileId, fsynced) // confirmed gone: drop from durable ledger glog.V(2).Infof("retry successful for %s after %d attempts", item.FileId, item.RetryCount) case deletionOutcomeNotFound: notFoundCount++ - f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted) - f.forgetDeletion(item.FileId) // already gone: drop from durable ledger + f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted) + f.forgetDeletion(item.FileId, fsynced) // already gone: drop from durable ledger case deletionOutcomeRetryable, deletionOutcomeNoResult: retryCount++ if outcome.status == deletionOutcomeNoResult { @@ -612,7 +634,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu // gave up: drop the record, unless the id was re-queued meanwhile // — a surviving newer record goes back through the hot queue. if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) { - f.queueDeletions(item.FileId) + f.queueDeletions(false, item.FileId) } glog.Warningf("permanent error on retry for %s after %d attempts: %s", item.FileId, item.RetryCount, outcome.errorMsg) } @@ -630,7 +652,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu } func (f *Filer) DeleteUncommittedChunks(ctx context.Context, chunks []*filer_pb.FileChunk) { - f.doDeleteChunks(ctx, chunks) + f.doDeleteChunks(ctx, chunks, false) } func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks []*filer_pb.FileChunk) { @@ -638,13 +660,13 @@ func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks if rule.DisableChunkDeletion { return } - f.doDeleteChunks(ctx, chunks) + f.doDeleteChunks(ctx, chunks, rule.GetFsync()) } -func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk) { +func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk, fsync bool) { for _, chunk := range chunks { if !chunk.IsChunkManifest { - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(fsync, chunk.GetFileIdString()) continue } dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk, f.MasterClient) @@ -652,15 +674,15 @@ func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk glog.V(0).InfofCtx(ctx, "failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) } for _, dChunk := range dataChunks { - f.queueDeletions(dChunk.GetFileIdString()) + f.queueDeletions(fsync, dChunk.GetFileIdString()) } - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(fsync, chunk.GetFileIdString()) } } func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) { for _, chunk := range chunks { - f.queueDeletions(chunk.GetFileIdString()) + f.queueDeletions(false, chunk.GetFileIdString()) } } diff --git a/weed/filer/filer_deletion_fsync_test.go b/weed/filer/filer_deletion_fsync_test.go new file mode 100644 index 0000000000..9b69eba869 --- /dev/null +++ b/weed/filer/filer_deletion_fsync_test.go @@ -0,0 +1,396 @@ +package filer + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strconv" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +type fsyncRecordingVolumeServer struct { + volume_server_pb.UnimplementedVolumeServerServer + mu sync.Mutex + fsync map[string][]bool + failOnce map[string]string + failCode int32 + inFlight func(req *volume_server_pb.BatchDeleteRequest) +} + +func (v *fsyncRecordingVolumeServer) BatchDelete(_ context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) { + if v.inFlight != nil { + v.inFlight(req) + } + v.mu.Lock() + defer v.mu.Unlock() + resp := &volume_server_pb.BatchDeleteResponse{} + for _, fid := range req.FileIds { + v.fsync[fid] = append(v.fsync[fid], req.Fsync) + if msg, ok := v.failOnce[fid]; ok { + delete(v.failOnce, fid) + code := v.failCode + if code == 0 { + code = http.StatusServiceUnavailable + } + resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{FileId: fid, Status: code, Error: msg}) + continue + } + resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{FileId: fid, Status: http.StatusAccepted}) + } + return resp, nil +} + +func (v *fsyncRecordingVolumeServer) calls(fid string) []bool { + v.mu.Lock() + defer v.mu.Unlock() + return append([]bool(nil), v.fsync[fid]...) +} + +func startFsyncRecordingVolumeServer(t *testing.T) (*fsyncRecordingVolumeServer, func([]string) (map[string]*operation.LookupResult, error)) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + v := &fsyncRecordingVolumeServer{fsync: map[string][]bool{}, failOnce: map[string]string{}} + srv := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(srv, v) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + + port := lis.Addr().(*net.TCPAddr).Port + location := operation.Location{Url: "127.0.0.1:" + strconv.Itoa(port), GrpcPort: port} + lookup := func(vids []string) (map[string]*operation.LookupResult, error) { + m := make(map[string]*operation.LookupResult, len(vids)) + for _, vid := range vids { + m[vid] = &operation.LookupResult{VolumeOrFileId: vid, Locations: []operation.Location{location}} + } + return m, nil + } + return v, lookup +} + +func newFsyncRuleTestFiler(t *testing.T) *Filer { + t.Helper() + f := newLedgerTestFiler(newStubFilerStore()) + f.GrpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials()) + f.FilerConf = NewFilerConf() + if err := f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/", Fsync: proto.Bool(true)}); err != nil { + t.Fatal(err) + } + return f +} + +func TestDeleteChunksFsyncFollowsPathRule(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: "3,01637037d6"}}) + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: "3,02637037d6"}}) + f.DeleteUncommittedChunks(ctx, []*filer_pb.FileChunk{{FileId: "3,03637037d6"}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + + for fid, want := range map[string]bool{"3,01637037d6": true, "3,02637037d6": false, "3,03637037d6": false} { + got := v.calls(fid) + if len(got) != 1 || got[0] != want { + t.Errorf("%s: BatchDelete fsync calls %v, want [%v]", fid, got, want) + } + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } + if ids := f.fsyncDeletionIds([]string{"3,01637037d6"}); len(ids) != 0 { + t.Errorf("fsync mark kept after the delete was confirmed: %v", ids) + } +} + +func TestDeleteChunksFsyncSurvivesRetry(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + fid := "4,01637037d6" + v.failOnce[fid] = "volume 4 is read only" + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: fid}}) + f.queueDeletions(false, fid) + f.FlushFileIdDeletionQueue(ctx, lookup) + if f.DeletionRetryQueue.Size() != 1 { + t.Fatalf("retry queue size %d, want 1", f.DeletionRetryQueue.Size()) + } + + f.DeletionRetryQueue.lock.Lock() + f.DeletionRetryQueue.itemIndex[fid].NextRetryAt = time.Now().Add(-time.Second) + f.DeletionRetryQueue.lock.Unlock() + ready, _ := f.DeletionRetryQueue.GetReadyItems(DeletionRetryBatchSize) + f.processRetryBatch(ready, lookup) + + if got := v.calls(fid); len(got) != 2 || !got[0] || !got[1] { + t.Fatalf("BatchDelete fsync calls %v, want [true true]", got) + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } +} + +// readLedgerAsOlderFiler decodes the ledger the way a filer without fsync +// marks does: a JSON array, or a {parts, gen} manifest over its part keys. +func readLedgerAsOlderFiler(t *testing.T, f *Filer) []string { + t.Helper() + ctx := context.Background() + key := f.deletionLedgerKey() + raw, err := f.Store.KvGet(ctx, []byte(key)) + if err != nil { + t.Fatalf("read ledger: %v", err) + } + var ids []string + if json.Unmarshal(raw, &ids) == nil { + return ids + } + var manifest struct { + Parts int `json:"parts"` + Gen int `json:"gen,omitempty"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatalf("ledger %q is neither an array nor a manifest: %v", raw, err) + } + for i := 0; i < manifest.Parts; i++ { + part, err := f.Store.KvGet(ctx, deletionLedgerGenPartKey(key, manifest.Gen, i)) + if err != nil { + t.Fatalf("read part %d: %v", i, err) + } + var partIds []string + if err := json.Unmarshal(part, &partIds); err != nil { + t.Fatalf("part %d: %v", i, err) + } + ids = append(ids, partIds...) + } + return ids +} + +func TestDeletionLedgerKeepsFsyncMarks(t *testing.T) { + for _, n := range []int{1, 4000} { + t.Run(strconv.Itoa(n), func(t *testing.T) { + withPersistedConfig(t, true) + store := newStubFilerStore() + f := newLedgerTestFiler(store) + var plain, synced []string + for i := 0; i < n; i++ { + plain = append(plain, fmt.Sprintf("7,%08xbeefcafe", i)) + synced = append(synced, fmt.Sprintf("8,%08xbeefcafe", i)) + } + f.queueDeletions(false, plain...) + f.queueDeletions(true, synced...) + f.snapshotDeletionLedger() + + if got := readLedgerAsOlderFiler(t, f); len(got) != 2*n || !contains(got, plain[0]) || !contains(got, synced[n-1]) { + t.Fatalf("older filer reads %d ids, want all %d", len(got), 2*n) + } + + f2 := newLedgerTestFiler(store) + if !f2.reloadDeletionLedger() { + t.Fatalf("ledger should reload") + } + if got := f2.pendingDeletionCount(); got != 2*n { + t.Fatalf("recovered %d ids, want %d", got, 2*n) + } + if got := f2.fsyncDeletionIds(append(append([]string(nil), plain...), synced...)); len(got) != n { + t.Fatalf("recovered %d fsync marks, want %d", len(got), n) + } + if got := f2.fsyncDeletionIds(plain); len(got) != 0 { + t.Fatalf("plain ids came back marked: %d", len(got)) + } + + for _, id := range synced { + f2.forgetDeletion(id, true) + } + f2.snapshotDeletionLedger() + f3 := newLedgerTestFiler(store) + f3.reloadDeletionLedger() + if got := f3.fsyncDeletionIds(synced); len(got) != 0 || f3.pendingDeletionCount() != n { + t.Fatalf("after forgetting the marked ids: %d marks, %d pending", len(got), f3.pendingDeletionCount()) + } + if n == 1 { + if _, ok := readPersistedLedger(t, f2); !ok { + t.Fatalf("a ledger without marks should be a single array again") + } + } + }) + } +} + +func TestDeletionLedgerClaimKeepsFsyncMarks(t *testing.T) { + withPersistedConfig(t, true) + store := newStubFilerStore() + fOld := newLedgerTestFiler(store) + fOld.Dlm = lock_manager.NewDistributedLockManager("filer-old:8888") + fOld.queueDeletions(false, "1,01") + fOld.queueDeletions(true, "1,02") + fOld.snapshotDeletionLedger() + + fNew := newLedgerTestFiler(store) + fNew.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999") + if !fNew.reloadDeletionLedger() { + t.Fatalf("reload should claim the stranded ledger") + } + if got := fNew.fsyncDeletionIds([]string{"1,01", "1,02"}); len(got) != 1 { + t.Fatalf("claimed fsync marks %v, want only 1,02", got) + } + + fNext := newLedgerTestFiler(store) + fNext.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999") + fNext.reloadDeletionLedger() + if _, ok := fNext.fsyncDeletionIds([]string{"1,02"})["1,02"]; !ok { + t.Fatalf("claimed fsync mark was not persisted under the new key") + } +} + +func TestQueueDeletionsMarksBeforePublishing(t *testing.T) { + f := newLedgerTestFiler(newStubFilerStore()) + const n = 20000 + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < n; i++ { + f.queueDeletions(true, fmt.Sprintf("9,%08xbeefcafe", i)) + } + }() + seen := 0 + for seen < n { + f.FileIdDeletionQueue.Consume(func(ids []string) { + marked := f.fsyncDeletionIds(ids) + for _, id := range ids { + if _, ok := marked[id]; !ok { + t.Errorf("%s reached the deletion queue before its fsync mark", id) + } + } + seen += len(ids) + }) + if t.Failed() { + break + } + } + <-done +} + +func TestRecursiveDeleteFsyncFollowsChildPaths(t *testing.T) { + store := newStubFilerStore() + f := newTestFiler(t, store, NewFilerRemoteStorage()) + for prefix, fsync := range map[string]bool{"/data/": true, "/mix/important": true, "/keep": true, "/keep/tmp-": false} { + if err := f.FilerConf.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: prefix, Fsync: proto.Bool(fsync)}); err != nil { + t.Fatal(err) + } + } + ctx := context.Background() + for _, dir := range []string{"/data", "/data/sub", "/mix", "/keep"} { + if err := f.CreateEntry(ctx, &Entry{FullPath: util.FullPath(dir), Attr: Attr{Mode: os.ModeDir | 0755}}, nil, false, false, nil, true, 255); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + "/data/a": "5,01637037d6", + "/data/sub/b": "5,02637037d6", + "/mix/important.txt": "5,03637037d6", + "/mix/other.txt": "5,04637037d6", + "/keep/y": "5,05637037d6", + "/keep/tmp-x": "5,06637037d6", + } + for path, fid := range files { + entry := &Entry{FullPath: util.FullPath(path), Attr: Attr{Mode: 0644, Mtime: time.Now()}, Chunks: []*filer_pb.FileChunk{{FileId: fid}}} + if err := f.CreateEntry(ctx, entry, nil, false, false, nil, true, 255); err != nil { + t.Fatal(err) + } + } + + for _, dir := range []string{"/data", "/mix", "/keep"} { + if err := f.DeleteEntryMetaAndData(ctx, util.FullPath(dir), true, false, true, false, nil, time.Time{}); err != nil { + t.Fatal(err) + } + } + + if n := f.pendingDeletionCount(); n != len(files) { + t.Fatalf("pending deletions %d, want %d", n, len(files)) + } + var fids []string + for _, fid := range files { + fids = append(fids, fid) + } + marked := f.fsyncDeletionIds(fids) + for path, fid := range files { + _, got := marked[fid] + if want := path != "/mix/other.txt" && path != "/keep/tmp-x"; got != want { + t.Errorf("%s: fsync mark %v, want %v", path, got, want) + } + } +} + +func TestPlainDeleteKeepsANewerFsyncMark(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + fid := "6,01637037d6" + var once sync.Once + v.inFlight = func(*volume_server_pb.BatchDeleteRequest) { + once.Do(func() { + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: fid}}) + }) + } + + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: fid}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + if _, ok := f.fsyncDeletionIds([]string{fid})[fid]; !ok || f.pendingDeletionCount() != 1 { + t.Fatalf("a plain delete dropped the fsync request queued while it ran") + } + + f.FlushFileIdDeletionQueue(ctx, lookup) + if got := v.calls(fid); len(got) != 2 || got[0] || !got[1] { + t.Fatalf("BatchDelete fsync calls %v, want [false true]", got) + } + if n := f.pendingDeletionCount(); n != 0 { + t.Errorf("pending deletions %d, want 0", n) + } +} + +func TestDurableDeleteRetriesAServerError(t *testing.T) { + withPersistedConfig(t, false) + v, lookup := startFsyncRecordingVolumeServer(t) + v.failCode = http.StatusInternalServerError + f := newFsyncRuleTestFiler(t) + ctx := context.Background() + durable, plain := "7,01637037d6", "7,02637037d6" + v.failOnce[durable] = "sync /data/7.dat: input/output error" + v.failOnce[plain] = "sync /data/7.dat: input/output error" + + f.DeleteChunks(ctx, util.FullPath("/buckets/b/obj"), []*filer_pb.FileChunk{{FileId: durable}}) + f.DeleteChunks(ctx, util.FullPath("/other/file"), []*filer_pb.FileChunk{{FileId: plain}}) + f.FlushFileIdDeletionQueue(ctx, lookup) + + f.DeletionRetryQueue.lock.Lock() + _, durableQueued := f.DeletionRetryQueue.itemIndex[durable] + _, plainQueued := f.DeletionRetryQueue.itemIndex[plain] + f.DeletionRetryQueue.lock.Unlock() + if !durableQueued || plainQueued { + t.Fatalf("retry queued: durable %v, plain %v; want durable only", durableQueued, plainQueued) + } + if _, ok := f.fsyncDeletionIds([]string{durable})[durable]; !ok || f.pendingDeletionCount() != 1 { + t.Fatalf("the failed durable delete lost its pending record or fsync mark") + } +} diff --git a/weed/filer/filer_deletion_persist.go b/weed/filer/filer_deletion_persist.go index 33b2371cf4..c1c18c0fd3 100644 --- a/weed/filer/filer_deletion_persist.go +++ b/weed/filer/filer_deletion_persist.go @@ -163,16 +163,16 @@ func (f *Filer) signalLedgerFlush() { // pipeline. It keeps the in-memory hot queue AND the durable ledger in sync. // Safe on a zero-value Filer (tests build struct literals without NewFiler): // the map is lazily created and the mutex is zero-value friendly. -func (f *Filer) queueDeletions(fileIds ...string) { +// fsync marks the ids for a durable volume delete; the mark stays until the +// record is forgotten, so a later enqueue without fsync keeps it. +func (f *Filer) queueDeletions(fsync bool, fileIds ...string) { if len(fileIds) == 0 { return } - // Hot path: existing in-memory queue, unchanged. - f.FileIdDeletionQueue.EnQueue(fileIds...) - // Durable ledger: record what still needs deleting. Every enqueue bumps the // id's epoch so an expiry-forget carrying an older epoch cannot erase the - // re-queued intent. + // re-queued intent. The record goes in before the id reaches the hot queue, + // so the delete loop never sees an id without its epoch and fsync mark. f.deletionLedgerLock.Lock() if f.pendingDeletions == nil { f.pendingDeletions = make(map[string]uint64, len(fileIds)) @@ -187,8 +187,18 @@ func (f *Filer) queueDeletions(fileIds ...string) { if !exists { f.deletionLedgerDirty = true } + if fsync { + if f.fsyncDeletions == nil { + f.fsyncDeletions = make(map[string]struct{}) + } + if _, marked := f.fsyncDeletions[id]; !marked { + f.fsyncDeletions[id] = struct{}{} + f.deletionLedgerDirty = true + } + } } f.deletionLedgerLock.Unlock() + f.FileIdDeletionQueue.EnQueue(fileIds...) f.signalLedgerFlush() } @@ -208,17 +218,41 @@ func (f *Filer) deletionEpoch(fileId string) uint64 { return f.pendingDeletions[fileId] } +// fsyncDeletionIds returns the subset of fileIds whose deletes must be durable. +func (f *Filer) fsyncDeletionIds(fileIds []string) map[string]struct{} { + f.deletionLedgerLock.Lock() + defer f.deletionLedgerLock.Unlock() + if len(f.fsyncDeletions) == 0 { + return nil + } + var ids map[string]struct{} + for _, id := range fileIds { + if _, ok := f.fsyncDeletions[id]; ok { + if ids == nil { + ids = make(map[string]struct{}) + } + ids[id] = struct{}{} + } + } + return ids +} + // forgetDeletion removes a fileId from the ledger once its deletion is terminal // (deleted or already absent: the chunks are gone, so any pending record — even // a re-queued one — refers to work that no longer exists). Non-terminal // outcomes must NOT call this so the entry survives until confirmed. -func (f *Filer) forgetDeletion(fileId string) { +// fsynced reports whether the finished delete was durable; if not, a record +// marked for fsync after that delete began stays for its queued copy, whose +// durable delete syncs the tombstone. +func (f *Filer) forgetDeletion(fileId string, fsynced bool) { if fileId == "" { return } f.deletionLedgerLock.Lock() - if _, exists := f.pendingDeletions[fileId]; exists { + _, marked := f.fsyncDeletions[fileId] + if _, exists := f.pendingDeletions[fileId]; exists && (fsynced || !marked) { delete(f.pendingDeletions, fileId) + delete(f.fsyncDeletions, fileId) f.deletionLedgerDirty = true } f.deletionLedgerLock.Unlock() @@ -237,6 +271,7 @@ func (f *Filer) forgetDeletionEpoch(fileId string, epoch uint64) bool { f.deletionLedgerLock.Lock() if cur, exists := f.pendingDeletions[fileId]; exists && cur == epoch { delete(f.pendingDeletions, fileId) + delete(f.fsyncDeletions, fileId) f.deletionLedgerDirty = true removed = true } @@ -280,10 +315,14 @@ func (f *Filer) snapshotDeletionLedger() { for id := range f.pendingDeletions { ids = append(ids, id) } + fsyncIds := make([]string, 0, len(f.fsyncDeletions)) + for id := range f.fsyncDeletions { + fsyncIds = append(fsyncIds, id) + } f.deletionLedgerDirty = false f.deletionLedgerLock.Unlock() - if err := f.writeDeletionLedger(ids); err != nil { + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { glog.Warningf("failed to persist deletion ledger (%d ids): %v", len(ids), err) f.deletionLedgerLock.Lock() f.deletionLedgerDirty = true @@ -306,7 +345,9 @@ func (f *Filer) tryUnblockDeletionLedger() bool { // atomically; a multipart set writes a new generation's part keys first and // only then the manifest, so a crash or failed write leaves the previously // published generation readable. Orphaned parts are retried on the next write. -func (f *Filer) writeDeletionLedger(ids []string) error { +// fsyncIds go into the trailing fsyncParts parts of a manifest, which older +// filers read as plain ids. +func (f *Filer) writeDeletionLedger(ids, fsyncIds []string) error { ctx := context.Background() key := f.deletionLedgerKey() @@ -332,7 +373,7 @@ func (f *Filer) writeDeletionLedger(ids []string) error { size = 2 return nil } - for _, id := range ids { + add := func(id string) error { if need := len(id) + 3; len(batch) > 0 && size+need > deletionLedgerPartSize { if err := flush(); err != nil { return err @@ -340,14 +381,39 @@ func (f *Filer) writeDeletionLedger(ids []string) error { } batch = append(batch, id) size += len(id) + 3 + return nil } - if len(batch) > 0 || len(parts) == 0 { + fsyncSet := make(map[string]struct{}, len(fsyncIds)) + for _, id := range fsyncIds { + fsyncSet[id] = struct{}{} + } + for _, id := range ids { + if _, ok := fsyncSet[id]; ok { + continue + } + if err := add(id); err != nil { + return err + } + } + if len(batch) > 0 || (len(parts) == 0 && len(fsyncSet) == 0) { if err := flush(); err != nil { return err } } + plainParts := len(parts) + for id := range fsyncSet { + if err := add(id); err != nil { + return err + } + } + if len(batch) > 0 { + if err := flush(); err != nil { + return err + } + } + fsyncParts := len(parts) - plainParts - if len(parts) == 1 { + if len(parts) == 1 && fsyncParts == 0 { if err := f.Store.KvPut(ctx, []byte(key), parts[0]); err != nil { return err } @@ -365,10 +431,7 @@ func (f *Filer) writeDeletionLedger(ids []string) error { } wrote++ } - manifest, _ := json.Marshal(struct { - Parts int `json:"parts"` - Gen int `json:"gen,omitempty"` - }{len(parts), gen}) + manifest, _ := json.Marshal(deletionLedgerManifest{Parts: len(parts), Gen: gen, FsyncParts: fsyncParts}) if err := f.Store.KvPut(ctx, []byte(key), manifest); err != nil { f.markStaleLedgerParts(key, gen, wrote) return err @@ -523,42 +586,49 @@ func (f *Filer) pruneLedgerIndex(ctx context.Context, key string) { _ = f.Store.KvPut(ctx, []byte(KvKeyDeletionLedgerIndex), payload) } +type deletionLedgerManifest struct { + Parts int `json:"parts"` + Gen int `json:"gen,omitempty"` + FsyncParts int `json:"fsyncParts,omitempty"` +} + // readDeletionLedger reads the manifest key: a JSON array is the whole set; a // {"parts":N,"gen":G} manifest points at that generation's part keys. A part // the manifest references but the store lacks is corruption, not absence, so -// it surfaces as a wrapped error rather than ErrKvNotFound. -func (f *Filer) readDeletionLedger(key string) (ids []string, gen, parts int, err error) { +// it surfaces as a wrapped error rather than ErrKvNotFound. ids holds every +// id; fsyncIds repeats the ones from the trailing fsyncParts parts. +func (f *Filer) readDeletionLedger(key string) (ids, fsyncIds []string, gen, parts int, err error) { ctx := context.Background() payload, err := f.Store.KvGet(ctx, []byte(key)) if err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } if !bytes.HasPrefix(bytes.TrimSpace(payload), []byte("{")) { if err := json.Unmarshal(payload, &ids); err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } - return ids, 0, 0, nil - } - var manifest struct { - Parts int `json:"parts"` - Gen int `json:"gen,omitempty"` + return ids, nil, 0, 0, nil } + var manifest deletionLedgerManifest if err := json.Unmarshal(payload, &manifest); err != nil { - return nil, 0, 0, err + return nil, nil, 0, 0, err } for i := 0; i < manifest.Parts; i++ { partPayload, err := f.Store.KvGet(ctx, deletionLedgerGenPartKey(key, manifest.Gen, i)) if err != nil { - return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s unreadable: %w", i, key, err) + return nil, nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s unreadable: %w", i, key, err) } var part []string if err := json.Unmarshal(partPayload, &part); err != nil { - return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s corrupt: %w", i, key, err) + return nil, nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s corrupt: %w", i, key, err) } ids = append(ids, part...) + if i >= manifest.Parts-manifest.FsyncParts { + fsyncIds = append(fsyncIds, part...) + } } - return ids, manifest.Gen, manifest.Parts, nil + return ids, fsyncIds, manifest.Gen, manifest.Parts, nil } // recoverForeignDeletionLedgers runs when the scoped key is absent: the ledger @@ -568,18 +638,19 @@ func (f *Filer) readDeletionLedger(key string) (ids []string, gen, parts int, er // deleted only after that write succeeds. A live peer's ledger claimed here is // rewritten by the peer's next snapshot, so the ids only get processed twice — // deletions are not owner-specific. -func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err error) { +func (f *Filer) recoverForeignDeletionLedgers(key string) (ids, fsyncIds []string, err error) { ctx := context.Background() claimed := map[string]ledgerClaim{} // source key -> what was read - lIds, lGen, lParts, lErr := f.readDeletionLedger(KvKeyDeletionLedger) + lIds, lFsyncIds, lGen, lParts, lErr := f.readDeletionLedger(KvKeyDeletionLedger) switch { case lErr == nil: ids = append(ids, lIds...) + fsyncIds = append(fsyncIds, lFsyncIds...) claimed[KvKeyDeletionLedger] = ledgerClaim{lIds, lGen, lParts} f.loadStaleLedgerParts(ctx, KvKeyDeletionLedger) case lErr != ErrKvNotFound: - return nil, lErr + return nil, nil, lErr } // Any read failure aborts the whole claim so reload retries: skipping an @@ -587,36 +658,37 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err // the index is never searched again. indexKeys, idxErr := f.readLedgerIndex(ctx) if idxErr != nil && idxErr != ErrKvNotFound { - return nil, idxErr + return nil, nil, idxErr } for _, other := range indexKeys { if other == key { continue } - oIds, oGen, oParts, oErr := f.readDeletionLedger(other) + oIds, oFsyncIds, oGen, oParts, oErr := f.readDeletionLedger(other) if oErr == ErrKvNotFound { f.pruneLedgerIndex(ctx, other) continue } if oErr != nil { - return nil, fmt.Errorf("deletion ledger %s unreadable: %w", other, oErr) + return nil, nil, fmt.Errorf("deletion ledger %s unreadable: %w", other, oErr) } ids = append(ids, oIds...) + fsyncIds = append(fsyncIds, oFsyncIds...) claimed[other] = ledgerClaim{oIds, oGen, oParts} f.loadStaleLedgerParts(ctx, other) } if len(claimed) == 0 { - return nil, ErrKvNotFound + return nil, nil, ErrKvNotFound } - if err := f.writeDeletionLedger(ids); err != nil { - return nil, err + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { + return nil, nil, err } mergedMore := false for src, claim := range claimed { // A source republished since we read it belongs to a live filer // (or a racing claimer): merge the newer ids and leave it in place. - curIds, curGen, curParts, rerr := f.readDeletionLedger(src) + curIds, curFsyncIds, curGen, curParts, rerr := f.readDeletionLedger(src) if rerr != nil || curGen != claim.gen || curParts != claim.parts || !sameStringSet(curIds, claim.ids) { if rerr == nil { for _, id := range curIds { @@ -625,6 +697,12 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err mergedMore = true } } + for _, id := range curFsyncIds { + if !containsId(fsyncIds, id) { + fsyncIds = append(fsyncIds, id) + mergedMore = true + } + } } glog.V(0).Infof("deletion ledger %s changed while claiming; leaving it for its owner", src) continue @@ -637,11 +715,11 @@ func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err err if mergedMore { // Ids from a changed source are durable only under that source's key; // rewrite our ledger so they survive under ours too. - if err := f.writeDeletionLedger(ids); err != nil { - return nil, err + if err := f.writeDeletionLedger(ids, fsyncIds); err != nil { + return nil, nil, err } } - return ids, nil + return ids, fsyncIds, nil } func containsId(ids []string, id string) bool { @@ -692,10 +770,10 @@ func (f *Filer) reloadDeletionLedger() bool { } key := f.deletionLedgerKey() - ids, gen, parts, err := f.readDeletionLedger(key) + ids, fsyncIds, gen, parts, err := f.readDeletionLedger(key) stateWritten := false if err == ErrKvNotFound && key != KvKeyDeletionLedger { - ids, err = f.recoverForeignDeletionLedgers(key) + ids, fsyncIds, err = f.recoverForeignDeletionLedgers(key) stateWritten = err == nil } if err != nil { @@ -725,6 +803,14 @@ func (f *Filer) reloadDeletionLedger() bool { } } } + for _, id := range fsyncIds { + if _, exists := f.pendingDeletions[id]; exists { + if f.fsyncDeletions == nil { + f.fsyncDeletions = make(map[string]struct{}, len(fsyncIds)) + } + f.fsyncDeletions[id] = struct{}{} + } + } } if !stateWritten { f.deletionLedgerGen = gen @@ -751,7 +837,7 @@ func (f *Filer) reloadDeletionLedger() bool { // for peer meta-aggregation to settle. The ledger is deliberately NOT // cleared here: it shrinks only as the delete pipeline confirms each // id terminal, so a second crash mid-recovery replays everything. - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) glog.V(0).Infof("re-queued %d recovered pending deletions", len(ids)) }() return true diff --git a/weed/filer/filer_deletion_persist_test.go b/weed/filer/filer_deletion_persist_test.go index d9f82f54b7..3432a37921 100644 --- a/weed/filer/filer_deletion_persist_test.go +++ b/weed/filer/filer_deletion_persist_test.go @@ -64,7 +64,7 @@ func TestDeletionLedgerSnapshotAndRecover(t *testing.T) { store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("1,01", "1,02", "1,03") + f.queueDeletions(false, "1,01", "1,02", "1,03") f.snapshotDeletionLedger() persisted, ok := readPersistedLedger(t, f) @@ -81,7 +81,7 @@ func TestDeletionLedgerSnapshotAndRecover(t *testing.T) { } // Confirmed-gone for one id, then snapshot again. - f.forgetDeletion("1,02") + f.forgetDeletion("1,02", false) f.snapshotDeletionLedger() persisted, _ = readPersistedLedger(t, f) if len(persisted) != 2 { @@ -130,7 +130,7 @@ func TestDeletionLedgerRetryKeepsEntry(t *testing.T) { store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("9,01") + f.queueDeletions(false, "9,01") f.snapshotDeletionLedger() // A retryable outcome deliberately does NOT call forgetDeletion, so the id @@ -152,7 +152,7 @@ func TestDeletionLedgerDisabled(t *testing.T) { withPersistedConfig(t, true) store := newStubFilerStore() fOn := newLedgerTestFiler(store) - fOn.queueDeletions("5,01") + fOn.queueDeletions(false, "5,01") fOn.snapshotDeletionLedger() if _, ok := readPersistedLedger(t, fOn); !ok { t.Fatalf("setup: expected a persisted ledger with persist enabled") @@ -163,7 +163,7 @@ func TestDeletionLedgerDisabled(t *testing.T) { // 1. Snapshot does not touch KV (no new writes, no clear). f := newLedgerTestFiler(store) - f.queueDeletions("6,01") // in-memory only + f.queueDeletions(false, "6,01") // in-memory only f.snapshotDeletionLedger() persisted, ok := readPersistedLedger(t, f) if !ok { @@ -192,8 +192,8 @@ func TestDeletionLedgerScopedPerFiler(t *testing.T) { fB := newLedgerTestFiler(store) fB.Dlm = lock_manager.NewDistributedLockManager("filer-b:8888") - fA.queueDeletions("1,01") - fB.queueDeletions("2,02") + fA.queueDeletions(false, "1,01") + fB.queueDeletions(false, "2,02") fA.snapshotDeletionLedger() fB.snapshotDeletionLedger() @@ -231,7 +231,7 @@ func TestDeletionLedgerChunked(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() raw, err := f.Store.KvGet(context.Background(), []byte(f.deletionLedgerKey())) @@ -250,7 +250,7 @@ func TestDeletionLedgerChunked(t *testing.T) { // Forget everything; the next snapshot is a single value again and the // part keys are removed. for _, id := range ids { - f2.forgetDeletion(id) + f2.forgetDeletion(id, false) } f2.snapshotDeletionLedger() raw, err = f.Store.KvGet(context.Background(), []byte(f2.deletionLedgerKey())) @@ -274,7 +274,7 @@ func TestDeletionLedgerReadFailureBlocksPersistence(t *testing.T) { t.Fatalf("reload should report the ledger as unusable") } - f.queueDeletions("1,01") + f.queueDeletions(false, "1,01") f.snapshotDeletionLedger() if len(store.kv) != 0 { t.Fatalf("snapshot must not overwrite a ledger that was never read, wrote %v", store.kv) @@ -297,7 +297,7 @@ func TestDeletionLedgerRecoveryKeepsIdsPending(t *testing.T) { withPersistedConfig(t, true) store := newStubFilerStore() f := newLedgerTestFiler(store) - f.queueDeletions("1,01", "1,02") + f.queueDeletions(false, "1,01", "1,02") f.snapshotDeletionLedger() util.GetViper().Set("filer.deleteQueue.recoveryGrace", time.Hour) @@ -326,11 +326,11 @@ func TestDeletionLedgerZeroValueFiler(t *testing.T) { DeletionRetryQueue: NewDeletionRetryQueue(), deletionQuit: make(chan struct{}), } - f.queueDeletions("0,01") + f.queueDeletions(false, "0,01") if f.pendingDeletionCount() != 1 { t.Fatalf("zero-value filer should lazily init the ledger, got %d", f.pendingDeletionCount()) } - f.forgetDeletion("0,01") + f.forgetDeletion("0,01", false) if f.pendingDeletionCount() != 0 { t.Fatalf("forget on zero-value filer failed, got %d", f.pendingDeletionCount()) } @@ -352,7 +352,7 @@ func TestDeletionLedgerMissingPartBlocks(t *testing.T) { if f.reloadDeletionLedger() { t.Fatalf("a manifest referencing a missing part is corrupt, not absent") } - f.queueDeletions("9,99") + f.queueDeletions(false, "9,99") f.snapshotDeletionLedger() if got := string(store.kv[key]); got != `{"parts":2,"gen":4}` { t.Fatalf("unread ledger must not be overwritten, got %q", got) @@ -367,7 +367,7 @@ func TestDeletionLedgerAddressChangeClaim(t *testing.T) { store := newStubFilerStore() fOld := newLedgerTestFiler(store) fOld.Dlm = lock_manager.NewDistributedLockManager("filer-old:8888") - fOld.queueDeletions("1,01", "1,02") + fOld.queueDeletions(false, "1,01", "1,02") fOld.snapshotDeletionLedger() fNew := newLedgerTestFiler(store) @@ -403,14 +403,14 @@ func TestDeletionLedgerGenerationPublish(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() key := f.deletionLedgerKey() if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 1, 0)); err != nil { t.Fatalf("generation 1 part must exist, got %v", err) } - f.queueDeletions("8,08") + f.queueDeletions(false, "8,08") f.snapshotDeletionLedger() if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 2, 0)); err != nil { t.Fatalf("generation 2 part must exist, got %v", err) @@ -439,18 +439,18 @@ func TestDeletionLedgerStalePartsRetried(t *testing.T) { for i := 0; i < 4000; i++ { ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i)) } - f.queueDeletions(ids...) + f.queueDeletions(false, ids...) f.snapshotDeletionLedger() key := f.deletionLedgerKey() store.kvDeleteErr = errors.New("delete down") - f.queueDeletions("8,08") + f.queueDeletions(false, "8,08") f.snapshotDeletionLedger() if len(f.deletionLedgerStale) == 0 { t.Fatalf("failed part deletes must be tracked for retry") } store.kvDeleteErr = nil - f.queueDeletions("8,09") + f.queueDeletions(false, "8,09") f.snapshotDeletionLedger() if len(f.deletionLedgerStale) != 0 { t.Fatalf("stale parts must flush once deletes work, got %v", f.deletionLedgerStale) @@ -556,9 +556,9 @@ func TestForgetDeletionEpochSkipsNewer(t *testing.T) { withPersistedConfig(t, true) f := newLedgerTestFiler(newStubFilerStore()) - f.queueDeletions("1,01") + f.queueDeletions(false, "1,01") oldEpoch := f.deletionEpoch("1,01") - f.queueDeletions("1,01") // re-enqueue bumps the epoch + f.queueDeletions(false, "1,01") // re-enqueue bumps the epoch newEpoch := f.deletionEpoch("1,01") if oldEpoch == newEpoch { t.Fatalf("re-enqueue must bump the epoch") diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index 297f665150..d7b9974051 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -45,11 +45,12 @@ func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc. return } - return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, lookupFunc) + return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, false, lookupFunc) } -func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult { +// fsync asks each volume server to make the deletes durable before answering. +func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, fsync bool, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult { var ret []*volume_server_pb.DeleteResult @@ -117,7 +118,7 @@ func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.Di go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false) + resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false, fsync) }(server, fidList) } @@ -133,7 +134,7 @@ func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.Di // DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc // Returns individual results for each file ID. Check result.Error for per-file failures. -func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult { +func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool, fsync bool) []*volume_server_pb.DeleteResult { var ret []*volume_server_pb.DeleteResult @@ -142,6 +143,7 @@ func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerA req := &volume_server_pb.BatchDeleteRequest{ FileIds: fileIds, SkipCookieCheck: !includeCookie, + Fsync: fsync, } resp, err := volumeServerClient.BatchDelete(ctx, req) diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 905813e43b..e56b324bc8 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -147,6 +147,7 @@ service VolumeServer { message BatchDeleteRequest { repeated string file_ids = 1; bool skip_cookie_check = 2; + bool fsync = 3; } message BatchDeleteResponse { diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 4e131ce276..d3d485f009 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -185,6 +185,7 @@ type BatchDeleteRequest struct { state protoimpl.MessageState `protogen:"open.v1"` FileIds []string `protobuf:"bytes,1,rep,name=file_ids,json=fileIds,proto3" json:"file_ids,omitempty"` SkipCookieCheck bool `protobuf:"varint,2,opt,name=skip_cookie_check,json=skipCookieCheck,proto3" json:"skip_cookie_check,omitempty"` + Fsync bool `protobuf:"varint,3,opt,name=fsync,proto3" json:"fsync,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -233,6 +234,13 @@ func (x *BatchDeleteRequest) GetSkipCookieCheck() bool { return false } +func (x *BatchDeleteRequest) GetFsync() bool { + if x != nil { + return x.Fsync + } + return false +} + type BatchDeleteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Results []*DeleteResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` @@ -7334,10 +7342,11 @@ const file_volume_server_proto_rawDesc = "" + "\x13volume_server.proto\x12\x10volume_server_pb\x1a\fremote.proto\"O\n" + "\x11VolumeServerState\x12 \n" + "\vmaintenance\x18\x01 \x01(\bR\vmaintenance\x12\x18\n" + - "\aversion\x18\x02 \x01(\rR\aversion\"[\n" + + "\aversion\x18\x02 \x01(\rR\aversion\"q\n" + "\x12BatchDeleteRequest\x12\x19\n" + "\bfile_ids\x18\x01 \x03(\tR\afileIds\x12*\n" + - "\x11skip_cookie_check\x18\x02 \x01(\bR\x0fskipCookieCheck\"O\n" + + "\x11skip_cookie_check\x18\x02 \x01(\bR\x0fskipCookieCheck\x12\x14\n" + + "\x05fsync\x18\x03 \x01(\bR\x05fsync\"O\n" + "\x13BatchDeleteResponse\x128\n" + "\aresults\x18\x01 \x03(\v2\x1e.volume_server_pb.DeleteResultR\aresults\"\x83\x01\n" + "\fDeleteResult\x12\x17\n" + diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index f15bdc8dc1..4092e5a0e8 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -3168,7 +3168,7 @@ func (s3a *S3ApiServer) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) { } // Attempt deletion using the operation package's batch delete with custom lookup - deleteResults := operation.DeleteFileIdsWithLookupVolumeId(context.Background(), s3a.option.GrpcDialOption, fileIds, lookupFunc) + deleteResults := operation.DeleteFileIdsWithLookupVolumeId(context.Background(), s3a.option.GrpcDialOption, fileIds, false, lookupFunc) // Log results - track successes and failures successCount := 0 diff --git a/weed/server/volume_grpc_batch_delete.go b/weed/server/volume_grpc_batch_delete.go index df996b38ab..c08abb5fc6 100644 --- a/weed/server/volume_grpc_batch_delete.go +++ b/weed/server/volume_grpc_batch_delete.go @@ -105,7 +105,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B // the delete cancels in-flight writes carrying this cookie n.Cookie = fidCookie } - if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, false); err != nil { + if size, err := vs.store.DeleteVolumeNeedle(volumeId, n, req.Fsync); err != nil { resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{ FileId: fid, Status: http.StatusInternalServerError, diff --git a/weed/server/volume_grpc_batch_delete_test.go b/weed/server/volume_grpc_batch_delete_test.go index 927b101f61..4f236245c2 100644 --- a/weed/server/volume_grpc_batch_delete_test.go +++ b/weed/server/volume_grpc_batch_delete_test.go @@ -2,15 +2,25 @@ package weed_server import ( "context" + "fmt" "net" + "net/http" + "sync/atomic" "testing" + "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" ) // TestBatchDelete_DeniesNonWhitelistedPeer verifies that BatchDelete refuses @@ -42,3 +52,46 @@ func TestBatchDelete_DeniesNonWhitelistedPeer(t *testing.T) { t.Fatalf("expected no per-file results on auth failure, got %d", len(resp.Results)) } } + +type syncCountingBackend struct { + backend.BackendStorageFile + syncs atomic.Int32 +} + +func (b *syncCountingBackend) Sync() error { + b.syncs.Add(1) + return b.BackendStorageFile.Sync() +} + +func TestBatchDeleteFsyncsOnlyWhenAsked(t *testing.T) { + for _, fsync := range []bool{false, true} { + t.Run(fmt.Sprint(fsync), func(t *testing.T) { + store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{t.TempDir()}, []int32{10}, []util.MinFreeSpace{{}}, "", storage.NeedleMapInMemory, + []types.DiskType{types.HardDriveType}, nil, 0, stats.DefaultDiskIOProbeConfig()) + t.Cleanup(store.Close) + require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0)) + n := &needle.Needle{Id: 1, Cookie: 0x1234, Data: []byte("chunk")} + n.Checksum = needle.NewCRC(n.Data) + _, err := store.WriteVolumeNeedle(1, n, true, false) + require.NoError(t, err) + + v := store.GetVolume(1) + counting := &syncCountingBackend{BackendStorageFile: v.DataBackend} + v.DataBackend = counting + + resp, err := (&VolumeServer{store: store}).BatchDelete(context.Background(), &volume_server_pb.BatchDeleteRequest{ + FileIds: []string{needle.NewFileIdFromNeedle(1, n).String()}, + Fsync: fsync, + }) + require.NoError(t, err) + require.Len(t, resp.Results, 1) + require.Equal(t, int32(http.StatusAccepted), resp.Results[0].Status, resp.Results[0].Error) + if fsync { + require.Positive(t, counting.syncs.Load(), "the tombstone was not synced") + } else { + require.Zero(t, counting.syncs.Load(), "a delete that did not ask for fsync paid for one") + } + }) + } +} diff --git a/weed/shell/command_fs_merge_volumes.go b/weed/shell/command_fs_merge_volumes.go index 5bf8c49d26..65f928f897 100644 --- a/weed/shell/command_fs_merge_volumes.go +++ b/weed/shell/command_fs_merge_volumes.go @@ -493,7 +493,7 @@ func deleteOrphanedNeedles(commandEnv *CommandEnv, entryPath util.FullPath, need continue } for _, loc := range locations { - results := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie) + results := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), loc.ServerAddress(), commandEnv.option.GrpcDialOption, fids, includeCookie, false) // Summarize per server: an unreachable volume server returns one // error per needle, which for manifest-heavy files can mean // hundreds of near-identical lines. Keep the first error as the diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index edd9416446..5c47cf86bd 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -769,7 +769,7 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me deleteResults := operation.DeleteFileIdsAtOneVolumeServer( context.Background(), pb.NewServerAddressFromDataNode(target.location.dataNode), - vcd.grpcDialOption(), fidList, false) + vcd.grpcDialOption(), fidList, false, false) // Check for errors in results for _, deleteResult := range deleteResults { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index a7d7ec918b..276130a2e5 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -1142,7 +1142,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - deleteResults := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), server, c.env.option.GrpcDialOption, fidList, false) + deleteResults := operation.DeleteFileIdsAtOneVolumeServer(context.Background(), server, c.env.option.GrpcDialOption, fidList, false, false) if deleteResults != nil { resultChan <- deleteResults }