Merge remote-tracking branch 'origin/master' into fix-delete-offline-replica

# Conflicts:
#	weed/storage/store_mark_readonly_can_delete_test.go

weed/shell/command_fs_distribute_chunks.go

@@ -23,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/protobuf/proto"
 )
 
 func init() {
@@ -116,6 +117,7 @@ func (c *commandFsDistributeChunks) Do(args []string, commandEnv *CommandEnv, wr
 	if err != nil {
 		return fmt.Errorf("lookup %s: %v", path, err)
 	}
+	readEntry := proto.Clone(entry).(*filer_pb.Entry)
 
 	chunks, hadManifest, err := resolveEntryDataChunks(commandEnv, entry)
 	if err != nil {
@@ -180,6 +182,11 @@ func (c *commandFsDistributeChunks) Do(args []string, commandEnv *CommandEnv, wr
 	fmt.Fprintf(writer, "\nExecuting redistribution...\n")
 	defer util_http.GetGlobalHttpClient().CloseIdleConnections()
 
+	readFids := make(map[string]bool, len(chunks))
+	for _, chunk := range chunks {
+		readFids[chunk.GetFileIdString()] = true
+	}
+
 	movedCount, err := executeChunkMoves(commandEnv, writer, moves, volumeInfoMap)
 	if err != nil {
 		return err
@@ -191,8 +198,14 @@ func (c *commandFsDistributeChunks) Do(args []string, commandEnv *CommandEnv, wr
 
 	finalChunks := chunks
 	if hadManifest {
-		// no chunk deleter here: a failed fold reports the blobs it saved
-		remanifested, manErr := filer.MaybeManifestize(newShellSaveAsChunk(commandEnv), nil, chunks)
+		dropSaved := func(saved []*filer_pb.FileChunk) {
+			var blobs []orphanedNeedle
+			for _, chunk := range saved {
+				blobs = append(blobs, orphanedNeedle{volumeId: chunk.GetFid().GetVolumeId(), fileId: chunk.GetFileIdString()})
+			}
+			deleteOrphanedNeedles(commandEnv, util.FullPath(path), blobs, true)
+		}
+		remanifested, manErr := filer.MaybeManifestize(newShellSaveAsChunk(commandEnv), dropSaved, chunks)
 		if manErr != nil {
 			fmt.Fprintf(writer, "WARNING: re-manifestize failed: %v. Writing flat chunk list.\n", manErr)
 		} else {
@@ -203,12 +216,24 @@ func (c *commandFsDistributeChunks) Do(args []string, commandEnv *CommandEnv, wr
 	entry.Chunks = finalChunks
 
 	fmt.Fprintf(writer, "\nUpdating filer metadata (%d chunks moved)...\n", movedCount)
+	var stale bool
 	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		return filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
-			Directory: dir,
-			Entry:     entry,
-		})
+		var uErr error
+		stale, uErr = updateEntryIfUnchanged(context.Background(), client, dir, entry, readEntry)
+		return uErr
 	})
+	if stale {
+		var copies []orphanedNeedle
+		for _, chunk := range slices.Concat(chunks, finalChunks) {
+			if fid := chunk.GetFileIdString(); !readFids[fid] {
+				readFids[fid] = true
+				copies = append(copies, orphanedNeedle{volumeId: chunk.GetFid().GetVolumeId(), fileId: fid})
+			}
+		}
+		deleteOrphanedNeedles(commandEnv, util.FullPath(path), copies, true)
+		fmt.Fprintf(writer, "%s was changed or deleted during redistribution; the filer kept that version and the moved copies were deleted.\n", path)
+		return err
+	}
 	if err != nil {
 		fmt.Fprintf(writer, "FAILED to update filer metadata: %v\n", err)
 		fmt.Fprintf(writer, "WARNING: New chunks were uploaded but metadata was not updated.\n")

weed/shell/command_fs_merge_volumes.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"sort"
 	"strconv"
@@ -21,6 +22,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"golang.org/x/exp/maps"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/seaweedfs/seaweedfs/weed/operation"
@@ -54,6 +57,9 @@ func (c *commandFsMergeVolumes) Help() string {
 	-toVolumeId accepts a comma-separated list. With -fromVolumeId, the source
 	chunks are distributed across the listed volumes by remaining capacity, so a
 	volume that does not fit into any single target can still be cleared.
+
+	-apply needs the shell lock (run "lock" first), so two runs never move the
+	same chunks at once.
 `
 }
 
@@ -72,6 +78,11 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 	if err = fsMergeVolumesCommand.Parse(args); err != nil {
 		return err
 	}
+	if *apply {
+		if err = commandEnv.confirmIsLocked(args); err != nil {
+			return err
+		}
+	}
 
 	dir := *dirArg
 	if dir != "/" {
@@ -171,7 +182,16 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 			// look like mergeVolumes hadn't done anything. Track the old fids
 			// and delete them below after the filer update commits, so the
 			// filer never points at a fid we already deleted.
-			var movedSources []orphanedNeedle
+			var moved rewrittenNeedles
+			// The update only lands while the filer still holds the entry as
+			// the walk read it, so keep that copy before the first chunk is
+			// rewritten in place.
+			var readEntry *filer_pb.Entry
+			keepReadEntry := func() {
+				if readEntry == nil {
+					readEntry = proto.Clone(entry).(*filer_pb.Entry)
+				}
+			}
 			for i, chunk := range entry.Chunks {
 				if chunk.IsChunkManifest {
 					if !c.manifestMayReferencePlan(plan, planCollections, needle.VolumeId(chunk.Fid.VolumeId)) {
@@ -190,14 +210,18 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 					if !changed || !*apply {
 						continue
 					}
+					keepReadEntry()
 					entry.Chunks[i] = newChunk
 					entryChanged = true
-					movedSources = append(movedSources, rewritten.sources...)
+					moved.sources = append(moved.sources, rewritten.sources...)
 					// The old manifest needle is always orphaned when we
 					// replace it with a freshly uploaded one, even when the
 					// rewrite was triggered by sub-chunk moves rather than the
 					// manifest volume itself being in the plan.
-					movedSources = append(movedSources, orphanedNeedle{volumeId: oldManifestVid, fileId: oldManifestFid})
+					moved.sources = append(moved.sources, orphanedNeedle{volumeId: oldManifestVid, fileId: oldManifestFid})
+					moved.targets = append(moved.targets, rewritten.targets...)
+					moved.targets = append(moved.targets, orphanedNeedle{volumeId: newChunk.Fid.VolumeId, fileId: newChunk.GetFileIdString()})
+					moved.reservations = append(moved.reservations, rewritten.reservations...)
 					continue
 				}
 
@@ -206,6 +230,7 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 					continue
 				}
 				seen.record(chunkVolumeId)
+				keepReadEntry()
 
 				oldFid := chunk.GetFileIdString()
 				oldVid := chunk.Fid.VolumeId
@@ -224,20 +249,29 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 					continue
 				}
 				entryChanged = true
-				movedSources = append(movedSources, orphanedNeedle{volumeId: oldVid, fileId: oldFid})
+				moved.sources = append(moved.sources, orphanedNeedle{volumeId: oldVid, fileId: oldFid})
+				moved.targets = append(moved.targets, orphanedNeedle{volumeId: uint32(toVolumeId), fileId: chunk.GetFileIdString()})
+				moved.reservations = append(moved.reservations, reservation{src: chunkVolumeId, dst: toVolumeId, size: chunk.Size})
 			}
 			if entryChanged {
-				if uErr := filer_pb.UpdateEntry(context.Background(), filerClient, &filer_pb.UpdateEntryRequest{
-					Directory: string(parentPath),
-					Entry:     entry,
-				}); uErr != nil {
+				stale, uErr := updateEntryIfUnchanged(context.Background(), filerClient, string(parentPath), entry, readEntry)
+				if stale {
+					fmt.Printf("skip %s: changed during merge\n", entryPath)
+					moved.targets = unreferencedTargets(context.Background(), filerClient, lookupFn, parentPath, entry.Name, moved.targets)
+					moved.abandon(commandEnv, entryPath, plan)
+					if len(entry.HardLinkId) > 0 {
+						processedHardLinks.Delete(string(entry.HardLinkId))
+					}
+					return nil
+				}
+				if uErr != nil {
 					fmt.Printf("failed to update %s: %v\n", entryPath, uErr)
 					// Filer still references the source fids. Deleting them
 					// now would lose data — abandon the cleanup for this
 					// entry and let fsck reconcile later.
 					return nil
 				}
-				deleteOrphanedNeedles(commandEnv, entryPath, movedSources, false)
+				deleteOrphanedNeedles(commandEnv, entryPath, moved.sources, false)
 			}
 			return nil
 		})
@@ -313,6 +347,61 @@ type rewrittenNeedles struct {
 	reservations []reservation
 }
 
+// abandon drops a rewrite the filer never took: the copies on the target
+// volumes belong to nobody, and the plan gets back the room they held, or a
+// multi-target merge keeps counting bytes it just deleted and skips later
+// chunks for no reason.
+func (r rewrittenNeedles) abandon(commandEnv *CommandEnv, entryPath util.FullPath, plan *mergePlan) {
+	deleteOrphanedNeedles(commandEnv, entryPath, r.targets, true)
+	for _, res := range r.reservations {
+		plan.release(res.src, res.dst, res.size)
+	}
+}
+
+// updateEntryIfUnchanged writes entry back only while the filer still holds
+// readEntry. An unconditional write would revert a change made since the read
+// and the filer would delete the newer chunks as garbage. stale reports that
+// the filer refused because the entry was changed or deleted, so nothing was
+// written and the copies made for this update are referenced by nothing.
+func updateEntryIfUnchanged(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string, entry, readEntry *filer_pb.Entry) (stale bool, err error) {
+	err = filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
+		Directory: dir,
+		Entry:     entry,
+		Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{
+			{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: readEntry},
+		}},
+	})
+	switch status.Code(err) {
+	case codes.FailedPrecondition, codes.NotFound:
+		return true, err
+	}
+	return false, err
+}
+
+// unreferencedTargets drops the copies the filer's current entry points at.
+// moveChunk keeps the key and cookie, so an overlapping run that moved the same
+// chunk to the same volume wrote the same fid, and its committed update is what
+// refused this one. When the entry cannot be read, no copy is reported.
+func unreferencedTargets(ctx context.Context, client filer_pb.SeaweedFilerClient, lookupFn wdclient.LookupFileIdFunctionType, dir util.FullPath, name string, targets []orphanedNeedle) []orphanedNeedle {
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{Directory: string(dir), Name: name})
+	if errors.Is(err, filer_pb.ErrNotFound) {
+		return targets
+	}
+	if err != nil {
+		return nil
+	}
+	chunks := resp.Entry.GetChunks()
+	dataChunks, manifestChunks, err := filer.ResolveChunkManifest(ctx, lookupFn, chunks, 0, math.MaxInt64, nil)
+	if err != nil {
+		return nil
+	}
+	referenced := make(map[string]bool)
+	for _, chunk := range slices.Concat(chunks, dataChunks, manifestChunks) {
+		referenced[chunk.GetFileIdString()] = true
+	}
+	return slices.DeleteFunc(targets, func(t orphanedNeedle) bool { return referenced[t.fileId] })
+}
+
 // deleteOrphanedNeedles fans out BatchDelete RPCs to every replica of each
 // needle's volume. Errors are logged but never returned — the data is already
 // orphan at this point, so a failed cleanup just leaves work for a later fsck.
@@ -740,16 +829,8 @@ func (c *commandFsMergeVolumes) rewriteManifestChunk(
 
 	var rewritten rewrittenNeedles
 	// Every path out of here that is not the successful return abandons the
-	// rewrite with the filer still pointing at the old manifest, so whatever
-	// already landed on the target volumes belongs to nobody. Give the plan
-	// back the room those copies were holding, or a multi-target merge keeps
-	// counting bytes it just deleted and skips later chunks for no reason.
-	abandon := func() {
-		deleteOrphanedNeedles(commandEnv, entryPath, rewritten.targets, true)
-		for _, r := range rewritten.reservations {
-			plan.release(r.src, r.dst, r.size)
-		}
-	}
+	// rewrite with the filer still pointing at the old manifest.
+	abandon := func() { rewritten.abandon(commandEnv, entryPath, plan) }
 	anySubChanged := false
 	for i, sub := range subChunks {
 		if sub.IsChunkManifest {

weed/shell/command_fs_merge_volumes_test.go

@@ -1,12 +1,22 @@
 package shell
 
 import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 func newMergeCmd(limitMB uint64, vols ...*master_pb.VolumeInformationMessage) *commandFsMergeVolumes {
@@ -394,3 +404,112 @@ func TestWarnUnreferencedSources_ConcurrentRecording(t *testing.T) {
 		t.Fatalf("expected no warning when all needles were seen, got %q", sb.String())
 	}
 }
+
+type conditionalUpdateFiler struct {
+	filer_pb.SeaweedFilerClient
+	err       error
+	req       *filer_pb.UpdateEntryRequest
+	current   *filer_pb.Entry
+	lookupErr error
+	lookupReq *filer_pb.LookupDirectoryEntryRequest
+}
+
+func (s *conditionalUpdateFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.lookupReq = req
+	if s.lookupErr != nil {
+		return nil, s.lookupErr
+	}
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: s.current}, nil
+}
+
+func (s *conditionalUpdateFiler) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
+	s.req = req
+	if s.err != nil {
+		return nil, s.err
+	}
+	return &filer_pb.UpdateEntryResponse{}, nil
+}
+
+// The write-back is conditioned on the entry as it was read, and only a refusal
+// that proves nothing was written counts as stale: the caller then deletes the
+// copies it made, which would lose data if the update might have landed.
+func TestUpdateEntryIfUnchanged(t *testing.T) {
+	read := &filer_pb.Entry{Name: "f", Chunks: []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 3, FileKey: 1, Cookie: 7}}}}
+	moved := &filer_pb.Entry{Name: "f", Chunks: []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 9, FileKey: 1, Cookie: 7}}}}
+
+	for _, tc := range []struct {
+		name  string
+		err   error
+		stale bool
+	}{
+		{"committed", nil, false},
+		{"changed", status.Error(codes.FailedPrecondition, "precondition failed"), true},
+		{"deleted", status.Error(codes.NotFound, "not found"), true},
+		{"outcome unknown", status.Error(codes.Unavailable, "filer down"), false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fake := &conditionalUpdateFiler{err: tc.err}
+			stale, err := updateEntryIfUnchanged(context.Background(), fake, "/dir", moved, read)
+			if stale != tc.stale || (err != nil) != (tc.err != nil) {
+				t.Fatalf("stale=%v err=%v, want stale=%v err=%v", stale, err, tc.stale, tc.err)
+			}
+			clauses := fake.req.GetCondition().GetClauses()
+			if len(clauses) != 1 || clauses[0].Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL ||
+				!proto.Equal(clauses[0].ExpectedEntry, read) || !proto.Equal(fake.req.Entry, moved) {
+				t.Fatalf("request %v must write the moved entry conditioned on the read one", fake.req)
+			}
+		})
+	}
+}
+
+// A refused update may have been refused by an overlapping run that moved the
+// same chunk to the same volume: moveChunk keeps the key and cookie, so that
+// run's committed entry points at the very fid this run would delete.
+func TestUnreferencedTargets(t *testing.T) {
+	committed := orphanedNeedle{volumeId: 9, fileId: "9,0101000007"}
+	own := orphanedNeedle{volumeId: 9, fileId: "9,0202000007"}
+	manifestCopy := orphanedNeedle{volumeId: 9, fileId: "9,0303000007"}
+	targets := func() []orphanedNeedle { return []orphanedNeedle{committed, own, manifestCopy} }
+	direct := &filer_pb.Entry{Name: "f", Chunks: []*filer_pb.FileChunk{{FileId: committed.fileId}}}
+
+	blob, err := proto.Marshal(&filer_pb.FileChunkManifest{Chunks: []*filer_pb.FileChunk{{FileId: committed.fileId, Size: 10}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	volumeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(blob)
+	}))
+	defer volumeServer.Close()
+	lookupFn := func(_ context.Context, fileId string) ([]string, error) {
+		if fileId != manifestCopy.fileId {
+			return nil, fmt.Errorf("volume of %s unavailable", fileId)
+		}
+		return []string{volumeServer.URL + "/" + fileId}, nil
+	}
+	viaManifest := &filer_pb.Entry{Name: "f", Chunks: []*filer_pb.FileChunk{{FileId: manifestCopy.fileId, IsChunkManifest: true, Size: 10}}}
+	unreadableManifest := &filer_pb.Entry{Name: "f", Chunks: []*filer_pb.FileChunk{{FileId: "9,0404000007", IsChunkManifest: true, Size: 10}}}
+
+	for _, tc := range []struct {
+		name      string
+		current   *filer_pb.Entry
+		lookupErr error
+		want      []orphanedNeedle
+	}{
+		{"another run committed one copy", direct, nil, []orphanedNeedle{own, manifestCopy}},
+		{"another run committed copies through a manifest", viaManifest, nil, []orphanedNeedle{own}},
+		{"manifest unreadable", unreadableManifest, nil, nil},
+		{"entry deleted", direct, status.Error(codes.NotFound, "not found"), targets()},
+		{"entry unreadable", direct, status.Error(codes.Unavailable, "filer down"), nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fake := &conditionalUpdateFiler{current: tc.current, lookupErr: tc.lookupErr}
+			got := unreferencedTargets(context.Background(), fake, lookupFn, "/dir", "f", targets())
+			if !slices.Equal(got, tc.want) {
+				t.Fatalf("got %v, want %v", got, tc.want)
+			}
+			if fake.lookupReq.GetDirectory() != "/dir" || fake.lookupReq.GetName() != "f" {
+				t.Fatalf("looked up %v, want /dir f", fake.lookupReq)
+			}
+		})
+	}
+}

weed/shell/command_fs_meta_change_volume_id.go

@@ -12,6 +12,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
 )
 
 func init() {
@@ -78,6 +79,7 @@ func (c *commandFsMetaChangeVolumeId) Do(args []string, commandEnv *CommandEnv,
 		return filer_pb.TraverseBfs(context.Background(), commandEnv, util.FullPath(*dir), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
 			if !entry.IsDirectory {
 				var hasChanges bool
+				var readEntry *filer_pb.Entry
 				for _, chunk := range entry.Chunks {
 					if chunk.IsChunkManifest {
 						fmt.Printf("Change volume id for large file is not implemented yet: %s/%s\n", parentPath, entry.Name)
@@ -85,6 +87,9 @@ func (c *commandFsMetaChangeVolumeId) Do(args []string, commandEnv *CommandEnv,
 					}
 					chunkVolumeId := chunk.Fid.VolumeId
 					if toVolumeId, found := mapping[needle.VolumeId(chunkVolumeId)]; found {
+						if readEntry == nil {
+							readEntry = proto.Clone(entry).(*filer_pb.Entry)
+						}
 						hasChanges = true
 						chunk.Fid.VolumeId = uint32(toVolumeId)
 						chunk.FileId = ""
@@ -93,10 +98,9 @@ func (c *commandFsMetaChangeVolumeId) Do(args []string, commandEnv *CommandEnv,
 				if hasChanges {
 					println("Updating", parentPath, entry.Name)
 					if *applyChanges {
-						if updateErr := filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
-							Directory: string(parentPath),
-							Entry:     entry,
-						}); updateErr != nil {
+						if stale, updateErr := updateEntryIfUnchanged(context.Background(), client, string(parentPath), entry, readEntry); stale {
+							fmt.Printf("skip %s/%s: changed since it was read\n", parentPath, entry.Name)
+						} else if updateErr != nil {
 							fmt.Printf("failed to update %s/%s: %v\n", parentPath, entry.Name, updateErr)
 						}
 					}

weed/shell/command_s3_versions_audit.go

@@ -13,6 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	s3_constants "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
 )
 
 func init() {
@@ -228,6 +229,7 @@ func healStrandedPointer(ctx context.Context, client filer_pb.SeaweedFilerClient
 	if entry.Extended == nil {
 		return nil
 	}
+	readEntry := proto.Clone(entry).(*filer_pb.Entry)
 	delete(entry.Extended, s3_constants.ExtLatestVersionIdKey)
 	delete(entry.Extended, s3_constants.ExtLatestVersionFileNameKey)
 	// Also clear the cached list metadata so a stale size/mtime/etag
@@ -237,10 +239,9 @@ func healStrandedPointer(ctx context.Context, client filer_pb.SeaweedFilerClient
 	delete(entry.Extended, s3_constants.ExtLatestVersionETagKey)
 	delete(entry.Extended, s3_constants.ExtLatestVersionOwnerKey)
 	delete(entry.Extended, s3_constants.ExtLatestVersionIsDeleteMarker)
-	_, err := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
-		Directory: string(parentPath),
-		Entry:     entry,
-	})
+	// A version written since the audit read the directory has moved the
+	// pointer, and clearing it would hide that version.
+	_, err := updateEntryIfUnchanged(ctx, client, string(parentPath), entry, readEntry)
 	return err
 }
 

weed/shell/command_s3_versions_audit_test.go

@@ -0,0 +1,37 @@
+package shell
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	s3_constants "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+// The heal clears the pointer only while the directory still names the file
+// the audit found missing; a version written since then has moved it.
+func TestHealStrandedPointerIsConditional(t *testing.T) {
+	fake := &conditionalUpdateFiler{}
+	entry := &filer_pb.Entry{Name: "obj.versions", IsDirectory: true, Extended: map[string][]byte{
+		s3_constants.ExtLatestVersionIdKey:       []byte("v1"),
+		s3_constants.ExtLatestVersionFileNameKey: []byte("v_v1"),
+		"keep":                                   []byte("x"),
+	}}
+	if err := healStrandedPointer(context.Background(), fake, "/buckets/b", entry); err != nil {
+		t.Fatalf("heal: %v", err)
+	}
+
+	clauses := fake.req.GetCondition().GetClauses()
+	if len(clauses) != 1 || clauses[0].Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL {
+		t.Fatalf("heal must be conditioned on the audited entry, got %v", fake.req.GetCondition())
+	}
+	if string(clauses[0].ExpectedEntry.Extended[s3_constants.ExtLatestVersionIdKey]) != "v1" {
+		t.Fatalf("expected entry must hold the pointer the audit saw, got %v", clauses[0].ExpectedEntry.Extended)
+	}
+	written := fake.req.Entry.Extended
+	_, idKept := written[s3_constants.ExtLatestVersionIdKey]
+	_, nameKept := written[s3_constants.ExtLatestVersionFileNameKey]
+	if idKept || nameKept || string(written["keep"]) != "x" {
+		t.Fatalf("written entry must clear only the pointer, got %v", written)
+	}
+}

weed/shell/command_volume_fsck.go

@@ -105,6 +105,16 @@ func (c *commandVolumeFsck) Help() string {
 	walked directory by directory; there, avoid running with -reallyDeleteFromVolume
 	while files are being renamed or moved.
 
+	With -findMissingChunksInFiler, -volumeId may name a volume the master no
+	longer lists, to find the filer entries that still reference it. A volume is
+	also unlisted while its server is offline, so purging these entries with
+	-reallyDeleteFilerEntries needs -confirmMissingVolumeDeletion as well; pass
+	it only after checking that every expected volume server is online. A purge
+	removes the whole filer entry, including its chunks on other volumes.
+	A -collection filter cannot be checked against an unlisted volume. With a
+	filter, a purge only reports references to unlisted volumes, and it refuses
+	to run when -volumeId names one.
+
 `
 }
 
@@ -123,6 +133,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
 	applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer. Currently this only works with default filerGroup.")
 	c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging")
 	purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, "<expert only!> delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler")
+	confirmMissingVolumeDeletion := fsckCommand.Bool("confirmMissingVolumeDeletion", false, "with -reallyDeleteFilerEntries, confirm that the -volumeId volumes the master does not list are retired")
 	c.purgeConcurrency = fsckCommand.Int("purgeConcurrency", 16, "parallel filer deletes for -reallyDeleteFilerEntries")
 	tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files")
 	cutoffTimeAgo := fsckCommand.Duration("cutoffTimeAgo", 5*time.Hour, "only include entries on volume servers before this cutoff time to check orphan chunks")
@@ -194,28 +205,26 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
 		}
 	}
 
-	// If the operator specified -volumeId, reject unknown ids up front instead
-	// of silently filtering them out and reporting "no orphan data". A silent
-	// success on a missing volume looks identical to a clean volume and hides
-	// typos, already-deleted volumes, and stale scripts.
 	if len(c.volumeIds) > 0 {
-		known := make(map[uint32]bool)
-		for _, vidMap := range dataNodeVolumeIdToVInfo {
-			for vid := range vidMap {
-				known[vid] = true
-			}
-		}
-		var missing []uint32
-		for vid := range c.volumeIds {
-			if !known[vid] {
-				missing = append(missing, vid)
-			}
+		missing, err := validateSelectedVolumeIDs(c.volumeIds, c.knownVolumeIds, *c.findMissingChunksInFiler)
+		if err != nil {
+			return err
 		}
 		if len(missing) > 0 {
-			sort.Slice(missing, func(i, j int) bool { return missing[i] < missing[j] })
-			return fmt.Errorf("volume(s) not found on master: %v", missing)
+			if *purgeAbsent {
+				if err := validateMissingVolumeCollectionFilter(missing, c.collectionMatcher); err != nil {
+					return err
+				}
+			}
+			if err := validateMissingVolumePurgeConfirmation(missing, *purgeAbsent, *confirmMissingVolumeDeletion); err != nil {
+				return err
+			}
+			fmt.Fprintf(c.writer, "WARNING: volume(s) not found on master (deleted or temporarily unavailable); scanning filer references for selected volume(s): %v\n", missing)
 		}
 	}
+	if *c.findMissingChunksInFiler && *purgeAbsent && c.collectionMatcher != nil {
+		fmt.Fprintln(c.writer, "WARNING: collection cannot be verified for volumes absent from master; Filer entries referencing those volumes will be reported but not purged.")
+	}
 
 	var collectCutoffFromAtNs int64 = 0
 	if cutoffTimeAgo.Seconds() != 0 {
@@ -360,13 +369,15 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 				// unknown), warn, and keep going. The unresolved counter blocks
 				// any purge step downstream so we never delete a sub-chunk we
 				// couldn't account for.
-				fmt.Fprintf(c.writer, "WARNING: ResolveChunkManifest failed for %s: %v — recording top-level chunk fids only; purging will be disabled\n",
+				fmt.Fprintf(c.writer, "WARNING: ResolveChunkManifest failed for %s: %v — recording top-level chunk fids only; orphan purging will be disabled\n",
 					util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr)
 				c.unresolvedManifestEntries.Add(1)
 				dataChunks = entry.Entry.GetChunks()
 				manifestChunks = nil
 			}
 			dataChunks = append(dataChunks, manifestChunks...)
+			// One Filer entry can have many missing chunks; delete the path only once.
+			purgePathQueued := false
 			for _, chunk := range dataChunks {
 				if cutoffFromAtNs != 0 && chunk.ModifiedTsNs > cutoffFromAtNs {
 					continue
@@ -374,6 +385,17 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 				if collectModifyFromAtNs != 0 && chunk.ModifiedTsNs < collectModifyFromAtNs {
 					continue
 				}
+				selectedVolume := len(c.volumeIds) == 0
+				if len(c.volumeIds) > 0 {
+					_, selectedVolume = c.volumeIds[chunk.Fid.VolumeId]
+				}
+				purgePath := purgeAbsent &&
+					selectedVolume &&
+					!c.knownVolumeIds[chunk.Fid.VolumeId] &&
+					!purgePathQueued && c.collectionMatcher == nil
+				if purgePath {
+					purgePathQueued = true
+				}
 				select {
 				case outputChan <- &Item{
 					vid:        chunk.Fid.VolumeId,
@@ -381,6 +403,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 					cookie:     chunk.Fid.Cookie,
 					path:       util.NewFullPath(entry.Dir, entry.Entry.Name),
 					hardLinkId: entry.Entry.GetHardLinkId(),
+					purgePath:  purgePath,
 				}:
 				case <-ctx.Done():
 					return ctx.Err()
@@ -422,7 +445,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 						continue
 					}
 					fmt.Fprintf(c.writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
-					if purgeAbsent {
+					if purgeAbsent && i.purgePath {
 						fmt.Fprintf(c.writer, "deleting path %s after volume not found\n", i.path)
 						queuePurge(purgeItem{path: i.path, hardLinkId: i.hardLinkId})
 					}
@@ -644,6 +667,7 @@ type Item struct {
 	cookie     uint32
 	path       util.FullPath
 	hardLinkId []byte
+	purgePath  bool
 }
 
 // lockedWriter serializes concurrent writers — the traversal, the consumer,
@@ -814,7 +838,9 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
 		fmt.Fprintln(c.writer, "delete response Headers : ", resp.Header)
 	}
 
-	if resp.StatusCode < http.StatusBadRequest {
+	if resp.StatusCode >= http.StatusMultipleChoices {
+		fmt.Fprintf(c.writer, "DELETE %s failed: %s\n", path, resp.Status)
+	} else {
 		dir, _ := path.DirAndName()
 		c.purgedDirsLock.Lock()
 		c.purgedDirs[util.FullPath(dir)] = struct{}{}
@@ -1094,3 +1120,40 @@ func writeToFile(bytes []byte, fileName string) error {
 	_, err = dst.Write(bytes)
 	return err
 }
+
+// validateSelectedVolumeIDs rejects IDs the master does not list, unless the
+// reverse filer scan is looking for references to exactly those volumes.
+func validateSelectedVolumeIDs(requested, known map[uint32]bool, allowMissing bool) ([]uint32, error) {
+	var missing []uint32
+	for vid := range requested {
+		if !known[vid] {
+			missing = append(missing, vid)
+		}
+	}
+	if len(missing) == 0 {
+		return nil, nil
+	}
+	sort.Slice(missing, func(i, j int) bool { return missing[i] < missing[j] })
+	if !allowMissing {
+		return nil, fmt.Errorf("volume(s) not found on master: %v", missing)
+	}
+	return missing, nil
+}
+
+// validateMissingVolumeCollectionFilter fails closed because a deleted volume
+// has no master metadata from which fsck can verify its collection.
+func validateMissingVolumeCollectionFilter(missing []uint32, matcher *wildcard.CollectionMatcher) error {
+	if len(missing) == 0 || matcher == nil {
+		return nil
+	}
+	return fmt.Errorf("cannot apply -collection=%q to missing volume(s) %v: their collection is unknown", matcher.String(), missing)
+}
+
+// validateMissingVolumePurgeConfirmation requires operators to distinguish
+// permanent retirement from a master that temporarily lost an offline server.
+func validateMissingVolumePurgeConfirmation(missing []uint32, purgeAbsent, confirmed bool) error {
+	if len(missing) == 0 || !purgeAbsent || confirmed {
+		return nil
+	}
+	return fmt.Errorf("refusing to purge Filer entries for volume(s) absent from master: %v; verify all expected volumes are connected and pass -confirmMissingVolumeDeletion", missing)
+}

weed/shell/command_volume_fsck_live_test.go

@@ -0,0 +1,360 @@
+//go:build integration
+
+package shell
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Opt-in: runs report-only and confirmed-purge scans against real services
+// after a volume is deleted, and checks that a stopped server blocks the purge.
+func TestVolumeFsckFindsReferenceAfterRealVolumeDeletion(t *testing.T) {
+	weed := buildWeedForIntegration(t)
+	base := t.TempDir()
+	for _, name := range []string{"logs", "master", "volume", "filer"} {
+		if err := os.MkdirAll(filepath.Join(base, name), 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	portPairs := freeIntegrationPortPairs(t, 3)
+	masterHTTP, masterGRPC := portPairs[0][0], portPairs[0][1]
+	volumeHTTP, volumeGRPC := portPairs[1][0], portPairs[1][1]
+	filerHTTP, filerGRPC := portPairs[2][0], portPairs[2][1]
+	masterAddress := fmt.Sprintf("127.0.0.1:%d", masterHTTP)
+
+	startIntegrationProcess(t, base, weed, "master",
+		"master", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+		fmt.Sprintf("-port=%d", masterHTTP), fmt.Sprintf("-port.grpc=%d", masterGRPC),
+		"-mdir="+filepath.Join(base, "master"), "-defaultReplication=000", "-volumeSizeLimitMB=64")
+	waitIntegrationTCP(t, masterHTTP, filepath.Join(base, "logs", "master.log"))
+	waitIntegrationTCP(t, masterGRPC, filepath.Join(base, "logs", "master.log"))
+
+	volumeProcess := startIntegrationProcess(t, base, weed, "volume",
+		"volume", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+		fmt.Sprintf("-port=%d", volumeHTTP), fmt.Sprintf("-port.grpc=%d", volumeGRPC),
+		"-master="+masterAddress, "-dir="+filepath.Join(base, "volume"), "-max=10")
+	waitIntegrationTCP(t, volumeHTTP, filepath.Join(base, "logs", "volume.log"))
+	waitIntegrationTCP(t, volumeGRPC, filepath.Join(base, "logs", "volume.log"))
+	waitForMasterVolumeRegistration(t, masterHTTP, volumeHTTP, filepath.Join(base, "logs", "volume.log"))
+
+	startIntegrationProcess(t, base, weed, "filer",
+		"filer", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
+		fmt.Sprintf("-port=%d", filerHTTP), fmt.Sprintf("-port.grpc=%d", filerGRPC),
+		"-master="+masterAddress, "-defaultReplicaPlacement=000",
+		"-defaultStoreDir="+filepath.Join(base, "filer"))
+	waitIntegrationTCP(t, filerHTTP, filepath.Join(base, "logs", "filer.log"))
+	waitIntegrationTCP(t, filerGRPC, filepath.Join(base, "logs", "filer.log"))
+
+	filerURL := fmt.Sprintf("http://127.0.0.1:%d", filerHTTP)
+	request, err := http.NewRequest(http.MethodPut, filerURL+"/fsck-probe.txt", strings.NewReader("live integration data"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatalf("upload probe through Filer: %v", err)
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusAccepted {
+		t.Fatalf("Filer PUT status = %d, want 201 or 202", response.StatusCode)
+	}
+
+	filerAddress := fmt.Sprintf("127.0.0.1:%d", filerHTTP)
+	meta := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /fsck-probe.txt\nunlock\n")
+	fidPattern := regexp.MustCompile(`"fid"\s*:\s*\{\s*"volumeId"\s*:\s*(\d+)`)
+	match := fidPattern.FindStringSubmatch(meta)
+	if len(match) != 2 {
+		t.Fatalf("could not read chunk volume ID from Filer metadata: %s", meta)
+	}
+	volumeID := match[1]
+
+	// Delete only the physical volume. The Filer entry intentionally remains
+	// so fsck must discover a dangling reference rather than a missing file.
+	deleteOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
+		fmt.Sprintf("lock\nvolume.delete -node=127.0.0.1:%d -volumeId=%s\nunlock\n", volumeHTTP, volumeID))
+	recentOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
+		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -volumeId=%s\nunlock\n", volumeID))
+	if strings.Contains(recentOutput, "/fsck-probe.txt") {
+		t.Fatalf("default cutoff unexpectedly reported a just-written file: %q", recentOutput)
+	}
+
+	fsckOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
+		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -volumeId=%s -cutoffTimeAgo=0\nunlock\n", volumeID))
+	if !strings.Contains(fsckOutput, "scanning filer references for selected volume(s)") {
+		t.Fatalf("fsck did not treat the deleted ID as a missing-volume target; delete=%q fsck=%q", deleteOutput, fsckOutput)
+	}
+	if !strings.Contains(fsckOutput, "/fsck-probe.txt") || !strings.Contains(fsckOutput, "volume not found") {
+		t.Fatalf("fsck did not report the stale Filer reference: %q", fsckOutput)
+	}
+
+	scopedPurgeOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
+		"lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -collection=logs -cutoffTimeAgo=0\nunlock\n")
+	if !strings.Contains(scopedPurgeOutput, "collection cannot be verified for volumes absent from master") {
+		t.Fatalf("fsck did not warn that a missing volume cannot be collection-scoped: %q", scopedPurgeOutput)
+	}
+	if strings.Contains(scopedPurgeOutput, "deleting path /fsck-probe.txt after volume not found") {
+		t.Fatalf("collection-filtered fsck queued an unverified missing-volume path: %q", scopedPurgeOutput)
+	}
+
+	// Report-only fsck must not delete the stale Filer entry.
+	metaAfter := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /fsck-probe.txt\nunlock\n")
+	if !strings.Contains(metaAfter, "fsck-probe.txt") {
+		t.Fatalf("report-only fsck changed or removed the Filer entry: %q", metaAfter)
+	}
+	// An entry whose chunk manifest lived on the retired volume is unreadable too.
+	filerConn, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", filerGRPC), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("dial Filer gRPC: %v", err)
+	}
+	defer filerConn.Close()
+	filerClient := filer_pb.NewSeaweedFilerClient(filerConn)
+	volumeIDUint, err := strconv.ParseUint(volumeID, 10, 32)
+	if err != nil {
+		t.Fatalf("parse volume ID %q: %v", volumeID, err)
+	}
+	manifestChunk := &filer_pb.FileChunk{
+		Fid:             &filer_pb.FileId{VolumeId: uint32(volumeIDUint), FileKey: 987654321, Cookie: 1},
+		Size:            21,
+		IsChunkManifest: true,
+	}
+	manifestChunk.FileId = manifestChunk.GetFileIdString()
+	now := time.Now().Unix()
+	if err := filer_pb.CreateEntry(context.Background(), filerClient, &filer_pb.CreateEntryRequest{
+		Directory: "/",
+		Entry: &filer_pb.Entry{
+			Name:       "manifest-probe.txt",
+			Attributes: &filer_pb.FuseAttributes{FileSize: 21, FileMode: 0o644, Mtime: now, Crtime: now},
+			Chunks:     []*filer_pb.FileChunk{manifestChunk},
+		},
+	}); err != nil {
+		t.Fatalf("create unresolved-manifest probe: %v", err)
+	}
+
+	purgeOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
+		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -confirmMissingVolumeDeletion -volumeId=%s -cutoffTimeAgo=0\nunlock\n", volumeID))
+	if !strings.Contains(purgeOutput, "ResolveChunkManifest failed") {
+		t.Fatalf("fsck did not surface the unresolved manifest: %q", purgeOutput)
+	}
+	if !strings.Contains(purgeOutput, "deleting path /manifest-probe.txt") {
+		t.Fatalf("confirmed purge did not queue the entry whose manifest was on the retired volume: %q", purgeOutput)
+	}
+	manifestAfterPurge, err := http.Get(filerURL + "/manifest-probe.txt")
+	if err != nil {
+		t.Fatalf("read manifest probe after confirmed purge: %v", err)
+	}
+	manifestAfterPurge.Body.Close()
+	if manifestAfterPurge.StatusCode != http.StatusNotFound {
+		t.Fatalf("confirmed purge manifest probe GET status = %d, want 404", manifestAfterPurge.StatusCode)
+	}
+
+	if !strings.Contains(purgeOutput, "deleting path /fsck-probe.txt after volume not found") {
+		t.Fatalf("confirmed purge did not queue the direct missing-volume entry: %q", purgeOutput)
+	}
+	probeAfterPurge, err := http.Get(filerURL + "/fsck-probe.txt")
+	if err != nil {
+		t.Fatalf("read Filer probe after confirmed purge: %v", err)
+	}
+	probeAfterPurge.Body.Close()
+	if probeAfterPurge.StatusCode != http.StatusNotFound {
+		t.Fatalf("confirmed purge Filer GET status = %d, want 404", probeAfterPurge.StatusCode)
+	}
+	// A volume still listed by the master but whose server is unreachable is
+	// not proof that its references are stale. Purge must fail before touching
+	// the Filer entry when the volume index cannot be read.
+	offlineRequest, err := http.NewRequest(http.MethodPut, filerURL+"/offline-probe.txt", strings.NewReader("offline safety probe"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	offlineResponse, err := http.DefaultClient.Do(offlineRequest)
+	if err != nil {
+		t.Fatalf("upload offline safety probe: %v", err)
+	}
+	offlineResponse.Body.Close()
+	if offlineResponse.StatusCode != http.StatusCreated && offlineResponse.StatusCode != http.StatusAccepted {
+		t.Fatalf("offline safety PUT status = %d", offlineResponse.StatusCode)
+	}
+	offlineMeta := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /offline-probe.txt\nunlock\n")
+	offlineMatch := fidPattern.FindStringSubmatch(offlineMeta)
+	if len(offlineMatch) != 2 {
+		t.Fatalf("could not read offline probe volume ID: %s", offlineMeta)
+	}
+	if err := volumeProcess.Process.Kill(); err != nil {
+		t.Fatalf("stop volume server: %v", err)
+	}
+	waitIntegrationTCPClosed(t, volumeHTTP)
+
+	shellCtx, cancelShell := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancelShell()
+	fsckCmd := exec.CommandContext(shellCtx, weed, "shell", "-master="+masterAddress, "-filer="+filerAddress)
+	fsckCmd.Stdin = strings.NewReader(fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -volumeId=%s -cutoffTimeAgo=0\nunlock\n", offlineMatch[1]))
+	fsckBytes, fsckErr := fsckCmd.CombinedOutput()
+	offlineFsckOutput := string(fsckBytes)
+	if fsckErr == nil {
+		t.Fatalf("fsck unexpectedly purged metadata for a volume server outage: %q", offlineFsckOutput)
+	}
+	if !strings.Contains(offlineFsckOutput, "confirmMissingVolumeDeletion") && !strings.Contains(offlineFsckOutput, "failed to collect file ids") {
+		t.Fatalf("fsck failed for an unexpected reason: err=%v output=%q", fsckErr, offlineFsckOutput)
+	}
+	offlineMetaAfter := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /offline-probe.txt\nunlock\n")
+	if !strings.Contains(offlineMetaAfter, "offline-probe.txt") {
+		t.Fatalf("offline-volume fsck removed the live Filer entry: %q", offlineMetaAfter)
+	}
+}
+
+func buildWeedForIntegration(t *testing.T) string {
+	t.Helper()
+	// Reuse the CI/integration binary when supplied; otherwise build this checkout.
+	if binary := os.Getenv("WEED_BINARY"); binary != "" {
+		if _, err := os.Stat(binary); err != nil {
+			t.Fatalf("WEED_BINARY=%q is unavailable: %v", binary, err)
+		}
+		return binary
+	}
+	_, source, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("resolve integration test source path")
+	}
+	root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", ".."))
+	binary := filepath.Join(t.TempDir(), "weed")
+	cmd := exec.Command("go", "build", "-o", binary, "./weed")
+	cmd.Dir = root
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("build weed: %v\n%s", err, out)
+	}
+	return binary
+}
+
+// freeIntegrationPortPairs reserves HTTP ports with the CLI's conventional
+// HTTP+10000 gRPC mapping. The low range also keeps the derived ports valid
+// for weed shell's master/Filer client configuration.
+func freeIntegrationPortPairs(t *testing.T, count int) [][2]int {
+	t.Helper()
+	var listeners []net.Listener
+	defer func() {
+		for _, listener := range listeners {
+			listener.Close()
+		}
+	}()
+
+	pairs := make([][2]int, 0, count)
+	for httpPort := 20000; httpPort < 30000 && len(pairs) < count; httpPort++ {
+		grpcPort := httpPort + 10000
+		httpListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", httpPort))
+		if err != nil {
+			continue
+		}
+		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+		if err != nil {
+			httpListener.Close()
+			continue
+		}
+		listeners = append(listeners, httpListener, grpcListener)
+		pairs = append(pairs, [2]int{httpPort, grpcPort})
+	}
+	if len(pairs) != count {
+		t.Fatalf("reserved %d of %d required HTTP/gRPC port pairs", len(pairs), count)
+	}
+	return pairs
+}
+
+func startIntegrationProcess(t *testing.T, base, weed, name string, args ...string) *exec.Cmd {
+	t.Helper()
+	logFile, err := os.Create(filepath.Join(base, "logs", name+".log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmd := exec.Command(weed, args...)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		t.Fatalf("start %s: %v", name, err)
+	}
+	t.Cleanup(func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+		_ = cmd.Wait()
+		_ = logFile.Close()
+	})
+	return cmd
+}
+
+// waitForMasterVolumeRegistration prevents racing a successful HTTP listener
+// start against the volume server's first heartbeat to the master.
+func waitForMasterVolumeRegistration(t *testing.T, masterPort, volumePort int, logPath string) {
+	t.Helper()
+	deadline := time.Now().Add(20 * time.Second)
+	wantAddress := fmt.Sprintf("127.0.0.1:%d", volumePort)
+	for time.Now().Before(deadline) {
+		response, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/dir/status", masterPort))
+		if err == nil {
+			body, readErr := io.ReadAll(response.Body)
+			response.Body.Close()
+			if readErr == nil && bytes.Contains(body, []byte(wantAddress)) {
+				return
+			}
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	log, _ := os.ReadFile(logPath)
+	t.Fatalf("master did not register volume server %s; log:\n%s", wantAddress, bytes.TrimSpace(log))
+}
+
+func waitIntegrationTCPClosed(t *testing.T, port int) {
+	t.Helper()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 100*time.Millisecond)
+		if err != nil {
+			return
+		}
+		conn.Close()
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("service still listens on port %d after process was stopped", port)
+}
+
+func waitIntegrationTCP(t *testing.T, port int, logPath string) {
+	t.Helper()
+	deadline := time.Now().Add(20 * time.Second)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 200*time.Millisecond)
+		if err == nil {
+			conn.Close()
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	log, _ := os.ReadFile(logPath)
+	t.Fatalf("service did not listen on port %d; log:\n%s", port, bytes.TrimSpace(log))
+}
+
+func runIntegrationShell(t *testing.T, weed, master, filer, input string) string {
+	t.Helper()
+	cmd := exec.Command(weed, "shell", "-master="+master, "-filer="+filer)
+	cmd.Stdin = strings.NewReader(input)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("weed shell failed: %v\n%s", err, out)
+	}
+	return string(out)
+}

weed/shell/command_volume_fsck_missing_volume_test.go

@@ -0,0 +1,184 @@
+package shell
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
+)
+
+// A selected volume can already be absent from the master when fsck is used to
+// clean stale Filer references after physical volume deletion. Only that
+// explicitly selected missing ID should enter the purge queue.
+func TestVolumeFsckFindMissingFilerEntriesForDeletedSelectedVolume(t *testing.T) {
+	tempDir := t.TempDir()
+	chunk := func(volumeID uint32, fileKey uint64) *filer_pb.FileChunk {
+		return &filer_pb.FileChunk{
+			Fid:  &filer_pb.FileId{VolumeId: volumeID, FileKey: fileKey, Cookie: 1},
+			Size: 1,
+		}
+	}
+	stub := &stubFilerServer{entries: map[string]*filer_pb.Entry{
+		"/":            {IsDirectory: true},
+		"/archive":     {IsDirectory: true},
+		"/archive/old": {Chunks: []*filer_pb.FileChunk{chunk(99, 1), chunk(99, 4), chunk(100, 2), chunk(7, 5)}},
+		"/live":        {Chunks: []*filer_pb.FileChunk{chunk(7, 3)}},
+	}}
+
+	verbose := false
+	findMissing := true
+	var output bytes.Buffer
+	c := &commandVolumeFsck{
+		tempFolder:               tempDir,
+		verbose:                  &verbose,
+		findMissingChunksInFiler: &findMissing,
+		writer:                   &output,
+		env:                      startStubFiler(t, stub),
+		volumeIds:                map[uint32]bool{99: true},
+		knownVolumeIds:           map[uint32]bool{7: true},
+	}
+	currentVolumes := map[string]map[uint32]VInfo{"dn1": {7: {collection: "live"}}}
+	var purged []purgeItem
+	if err := c.collectFilerFileIdAndPaths(currentVolumes, true, func(item purgeItem) {
+		purged = append(purged, item)
+	}, 0, 0); err != nil {
+		t.Fatalf("collect filer references: %v", err)
+	}
+
+	if len(purged) != 1 || purged[0].path != util.FullPath("/archive/old") {
+		t.Fatalf("purged paths = %v, want only /archive/old for selected missing volume 99", purged)
+	}
+	// The same logical file also references a live volume. The scan preserves
+	// that reference while queuing the path only once for the missing chunks.
+	foundSurvivingChunk := false
+	if err := c.readFilerFileIdFile(7, func(_ types.NeedleId, itemPath util.FullPath, _ []byte) {
+		if itemPath == util.FullPath("/archive/old") {
+			foundSurvivingChunk = true
+		}
+	}); err != nil {
+		t.Fatalf("read surviving volume references: %v", err)
+	}
+	if !foundSurvivingChunk {
+		t.Fatal("the surviving chunk reference was not recorded for volume 7")
+	}
+	for _, line := range []string{
+		"99,100000001 /archive/old volume not found\n",
+		"99,400000001 /archive/old volume not found\n",
+	} {
+		if !strings.Contains(output.String(), line) {
+			t.Errorf("missing-volume report %q lacks %q", output.String(), line)
+		}
+	}
+	if strings.Contains(output.String(), "100,200000001") {
+		t.Errorf("missing-volume report %q includes unselected volume 100", output.String())
+	}
+}
+
+// Unknown IDs remain errors outside the reverse Filer scan, preserving typo detection.
+func TestVolumeFsckSelectedVolumeIDValidation(t *testing.T) {
+	known := map[uint32]bool{7: true}
+	requested := map[uint32]bool{99: true, 100: true}
+
+	if _, err := validateSelectedVolumeIDs(requested, known, false); err == nil {
+		t.Fatal("normal fsck accepted volume IDs not present on the master")
+	}
+	missing, err := validateSelectedVolumeIDs(requested, known, true)
+	if err != nil {
+		t.Fatalf("reverse Filer scan rejected missing volume IDs: %v", err)
+	}
+	if len(missing) != 2 || missing[0] != 99 || missing[1] != 100 {
+		t.Fatalf("missing IDs = %v, want sorted [99 100]", missing)
+	}
+}
+
+// A missing volume has no trustworthy collection metadata; a collection
+// restriction must fail closed instead of silently widening or narrowing scope.
+func TestValidateMissingVolumeCollectionFilter(t *testing.T) {
+	missing := []uint32{12}
+	compile := func(filter string) *wildcard.CollectionMatcher {
+		matcher, err := wildcard.CompileCollectionMatcher(filter)
+		if err != nil {
+			t.Fatalf("compile %q: %v", filter, err)
+		}
+		return matcher
+	}
+	for _, filter := range []string{"", "*", "ALL_COLLECTIONS", "EACH_COLLECTION"} {
+		if err := validateMissingVolumeCollectionFilter(missing, compile(filter)); err != nil {
+			t.Errorf("filter %q: %v", filter, err)
+		}
+	}
+	if err := validateMissingVolumeCollectionFilter(missing, compile("photos")); err == nil || !strings.Contains(err.Error(), "collection is unknown") {
+		t.Fatalf("expected an unknown-collection error, got %v", err)
+	}
+}
+
+// Missing IDs require an extra acknowledgement only when metadata deletion
+// is requested; read-only diagnosis and known-volume purges are unchanged.
+func TestValidateMissingVolumePurgeConfirmation(t *testing.T) {
+	missing := []uint32{12}
+	for _, test := range []struct {
+		name        string
+		missing     []uint32
+		purgeAbsent bool
+		confirmed   bool
+		wantError   bool
+	}{
+		{name: "read-only scan", missing: missing},
+		{name: "purge known volumes only", purgeAbsent: true},
+		{name: "confirmed missing volume purge", missing: missing, purgeAbsent: true, confirmed: true},
+		{name: "unconfirmed missing volume purge", missing: missing, purgeAbsent: true, wantError: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateMissingVolumePurgeConfirmation(test.missing, test.purgeAbsent, test.confirmed)
+			if (err != nil) != test.wantError {
+				t.Fatalf("validateMissingVolumePurgeConfirmation() error = %v, wantError %v", err, test.wantError)
+			}
+			if err != nil && !strings.Contains(err.Error(), "confirmMissingVolumeDeletion") {
+				t.Fatalf("error %q does not name the required acknowledgement flag", err)
+			}
+		})
+	}
+}
+
+func TestVolumeFsckPurgesEntryWhoseManifestIsOnMissingVolume(t *testing.T) {
+	manifest := &filer_pb.FileChunk{
+		Fid:             &filer_pb.FileId{VolumeId: 99, FileKey: 1, Cookie: 1},
+		Size:            1,
+		IsChunkManifest: true,
+	}
+	stub := &stubFilerServer{entries: map[string]*filer_pb.Entry{
+		"/":     {IsDirectory: true},
+		"/big":  {Chunks: []*filer_pb.FileChunk{manifest}},
+		"/live": {Chunks: []*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 7, FileKey: 2, Cookie: 1}, Size: 1}}},
+	}}
+
+	verbose := false
+	findMissing := true
+	var output bytes.Buffer
+	c := &commandVolumeFsck{
+		tempFolder:               t.TempDir(),
+		verbose:                  &verbose,
+		findMissingChunksInFiler: &findMissing,
+		writer:                   &output,
+		env:                      startStubFiler(t, stub),
+		volumeIds:                map[uint32]bool{99: true},
+		knownVolumeIds:           map[uint32]bool{7: true},
+	}
+	var purged []purgeItem
+	if err := c.collectFilerFileIdAndPaths(map[string]map[uint32]VInfo{"dn1": {7: {}}}, true, func(item purgeItem) {
+		purged = append(purged, item)
+	}, 0, 0); err != nil {
+		t.Fatalf("collect filer references: %v", err)
+	}
+
+	if !strings.Contains(output.String(), "ResolveChunkManifest failed for /big") {
+		t.Fatalf("manifest on the missing volume resolved unexpectedly: %q", output.String())
+	}
+	if len(purged) != 1 || purged[0].path != util.FullPath("/big") {
+		t.Fatalf("purged paths = %v, want /big, whose manifest is on missing volume 99", purged)
+	}
+}

weed/shell/command_volume_fsck_test.go

@@ -56,9 +56,10 @@ func TestVolumeFsckHttpDeleteRecordsParentDirectory(t *testing.T) {
 	defer filer.Close()
 
 	verbose := false
+	var output bytes.Buffer
 	c := &commandVolumeFsck{
 		verbose:    &verbose,
-		writer:     io.Discard,
+		writer:     &output,
 		purgedDirs: make(map[util.FullPath]struct{}),
 		env:        &CommandEnv{option: &ShellOptions{FilerAddress: pb.ServerAddress(strings.TrimPrefix(filer.URL, "http://"))}},
 	}
@@ -72,6 +73,9 @@ func TestVolumeFsckHttpDeleteRecordsParentDirectory(t *testing.T) {
 	if _, found := c.purgedDirs["/locked/dir"]; found {
 		t.Errorf("expected a rejected delete not to be recorded, got %v", c.purgedDirs)
 	}
+	if got := output.String(); got != "DELETE /locked/dir/file.txt failed: 403 Forbidden\n" {
+		t.Errorf("delete output = %q, want only the rejected path", got)
+	}
 }
 
 // stubFilerServer keeps a flat set of full paths and enforces the same delete