mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* volume: fsync a batch delete that asks for it BatchDeleteRequest gets an fsync field, and the Go and Rust volume servers pass it to the needle delete, which already orders the tombstone, .dat sync, publish and .idx sync for a durable delete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: pass an fsync flag to batch deletes DeleteFileIdsWithLookupVolumeId and DeleteFileIdsAtOneVolumeServer take an fsync argument and send it in the BatchDeleteRequest. Every caller passes false for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: fsync chunk deletes under an fsync path rule DeleteChunks passes the path rule's fsync flag to the deletion queue. The flag is kept with the pending record until the delete is confirmed, so retries and re-queues keep it, and the batch and retry loops send marked ids in a durable BatchDelete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep fsync marks in the deletion ledger Marked ids go into the trailing parts of a manifest ledger, counted by fsyncParts. Older filers read those parts as plain ids and still delete them; a reload or a claimed foreign ledger restores the marks. A ledger without marks keeps the single array form. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: record a queued deletion before the delete loop can see it queueDeletions published the ids to the hot queue before taking the ledger lock, so a concurrent flush could delete an id before its fsync mark landed and then forget the mark with the record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: fsync a folder delete's chunks by each child's path rule A recursive delete collected its file children's chunks and resolved the rule once for the folder, which misses a trailing-slash rule on the folder itself and file-prefix rules. Each file child's own rule now decides fsync; the folder rule still decides chunk deletion. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a newer fsync request when a plain delete finishes An id re-queued with fsync while a plain delete of it was in flight lost its mark when that delete succeeded, so the queued copy also ran without fsync. A plain attempt now leaves a marked record for its queued copy; a durable delete of an already-deleted needle still syncs the tombstone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: delete a folder's plain child chunks without fsync Children whose own rule turns fsync off were still deleted durably when the folder's rule asked for fsync. Only the children marked by their own rule now go out with fsync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: retry a durable chunk delete the volume server failed A volume server rolls back a durable delete it cannot sync and answers with a 5xx carrying the I/O error text, which the classifier took as permanent, so the record and its fsync mark were dropped while the needle stayed live. A 5xx answer to a durable delete is now retryable, within the usual attempt limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
3.3 KiB
Go
98 lines
3.3 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/peer"
|
|
"google.golang.org/grpc/status"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// TestBatchDelete_DeniesNonWhitelistedPeer verifies that BatchDelete refuses
|
|
// callers that fall outside the configured admin whitelist, matching the
|
|
// behaviour of the other destructive volume-server RPCs.
|
|
func TestBatchDelete_DeniesNonWhitelistedPeer(t *testing.T) {
|
|
vs := &VolumeServer{
|
|
guard: security.NewGuard([]string{"10.0.0.1"}, "", 0, "", 0),
|
|
}
|
|
|
|
ctx := peer.NewContext(context.Background(), &peer.Peer{
|
|
Addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12345},
|
|
})
|
|
|
|
resp, err := vs.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
|
|
FileIds: []string{"1,deadbeef"},
|
|
SkipCookieCheck: true,
|
|
})
|
|
if err == nil {
|
|
t.Fatalf("expected PermissionDenied, got nil error (resp=%+v)", resp)
|
|
}
|
|
if got := status.Code(err); got != codes.PermissionDenied {
|
|
t.Fatalf("expected codes.PermissionDenied, got %v (err=%v)", got, err)
|
|
}
|
|
if resp == nil {
|
|
t.Fatalf("expected non-nil response even on auth failure")
|
|
}
|
|
if len(resp.Results) != 0 {
|
|
t.Fatalf("expected no per-file results on auth failure, got %d", len(resp.Results))
|
|
}
|
|
}
|
|
|
|
type syncCountingBackend struct {
|
|
backend.BackendStorageFile
|
|
syncs atomic.Int32
|
|
}
|
|
|
|
func (b *syncCountingBackend) Sync() error {
|
|
b.syncs.Add(1)
|
|
return b.BackendStorageFile.Sync()
|
|
}
|
|
|
|
func TestBatchDeleteFsyncsOnlyWhenAsked(t *testing.T) {
|
|
for _, fsync := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(fsync), func(t *testing.T) {
|
|
store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
|
|
[]string{t.TempDir()}, []int32{10}, []util.MinFreeSpace{{}}, "", storage.NeedleMapInMemory,
|
|
[]types.DiskType{types.HardDriveType}, nil, 0, stats.DefaultDiskIOProbeConfig())
|
|
t.Cleanup(store.Close)
|
|
require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0))
|
|
n := &needle.Needle{Id: 1, Cookie: 0x1234, Data: []byte("chunk")}
|
|
n.Checksum = needle.NewCRC(n.Data)
|
|
_, err := store.WriteVolumeNeedle(1, n, true, false)
|
|
require.NoError(t, err)
|
|
|
|
v := store.GetVolume(1)
|
|
counting := &syncCountingBackend{BackendStorageFile: v.DataBackend}
|
|
v.DataBackend = counting
|
|
|
|
resp, err := (&VolumeServer{store: store}).BatchDelete(context.Background(), &volume_server_pb.BatchDeleteRequest{
|
|
FileIds: []string{needle.NewFileIdFromNeedle(1, n).String()},
|
|
Fsync: fsync,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, resp.Results, 1)
|
|
require.Equal(t, int32(http.StatusAccepted), resp.Results[0].Status, resp.Results[0].Error)
|
|
if fsync {
|
|
require.Positive(t, counting.syncs.Load(), "the tombstone was not synced")
|
|
} else {
|
|
require.Zero(t, counting.syncs.Load(), "a delete that did not ask for fsync paid for one")
|
|
}
|
|
})
|
|
}
|
|
}
|