Merge branch 'master' of https://github.com/seaweedfs/seaweedfs into nfs-enable-inode-index

weed/shell/command_fs_meta_save.go

@@ -20,6 +20,9 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
 )
 
 func init() {
@@ -152,6 +155,133 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
 // log; volume.fsck and fs.verify must leave it false so the live log-file
 // chunks are accounted for — otherwise fsck reports them as orphans and could
 // delete referenced needles.
+// doTraverseAndSaving feeds every entry at and below path through genFn and
+// saveFn: from one snapshot of the filer store when the filer can provide it
+// (so a file renamed during the traversal is not missed), by walking otherwise.
+func doTraverseAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error {
+	if used, err := doTraverseSnapshotAndSaving(filerClient, writer, path, verbose, skipSystemLog, genFn, saveFn); used || err != nil {
+		return err
+	}
+	return doTraverseBfsAndSaving(filerClient, writer, path, verbose, skipSystemLog, genFn, saveFn)
+}
+
+// doTraverseSnapshotAndSaving asks the filer to traverse path itself and
+// consumes the stream only when the filer serves it from a store snapshot
+// (filer_pb.TraverseModeHeaderKey). Otherwise it returns used=false having
+// read nothing, and the caller walks as before.
+func doTraverseSnapshotAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) (used bool, err error) {
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var excludedPrefixes []string
+	if skipSystemLog {
+		excludedPrefixes = append(excludedPrefixes, filer.SystemLogDir)
+	}
+
+	// The walk tolerates a path that does not exist (it saves nothing); the
+	// filer-side traversal reports it as an error, so let the walk have it.
+	// Any other failure must stop here: a walk that could then reach the filer
+	// reads renamed-away entries as absent, and fsck would purge their chunks.
+	if _, _, _, getErr := filer_pb.GetEntry(ctx, filerClient, util.FullPath(path)); getErr != nil {
+		if errors.Is(getErr, filer_pb.ErrNotFound) {
+			return false, nil
+		}
+		return false, fmt.Errorf("traverse %s: %w", path, getErr)
+	}
+
+	err = filerClient.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error {
+		ctx := metadata.AppendToOutgoingContext(ctx, filer_pb.TraverseModeHeaderKey, filer_pb.TraverseModeSnapshot)
+		stream, err := client.TraverseBfsMetadata(ctx, &filer_pb.TraverseBfsMetadataRequest{
+			Directory:        path,
+			ExcludedPrefixes: excludedPrefixes,
+		})
+		if err != nil {
+			return err
+		}
+		header, err := stream.Header()
+		if err != nil {
+			if status.Code(err) == codes.Unimplemented {
+				return nil // a filer without the RPC at all: walk
+			}
+			return fmt.Errorf("traverse %s: %w", path, err)
+		}
+		if modes := header.Get(filer_pb.TraverseModeHeaderKey); len(modes) == 0 || modes[0] != filer_pb.TraverseModeSnapshot {
+			// An older filer (no header), a store that cannot scan ("bfs"), or
+			// a stream that failed before any header, which Header() reports
+			// as no header and nil: only Recv tells, and a failure is no
+			// licence to walk instead.
+			if _, recvErr := stream.Recv(); recvErr != nil && !errors.Is(recvErr, io.EOF) && status.Code(recvErr) != codes.Unimplemented {
+				return fmt.Errorf("traverse %s: %w", path, recvErr)
+			}
+			return nil
+		}
+		used = true
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		outputChan := make(chan interface{}, 1024)
+		saveErrChan := make(chan error, 1)
+		go func() {
+			saveErr := saveFn(outputChan)
+			saveErrChan <- saveErr
+			if saveErr != nil {
+				cancel()
+			}
+			wg.Done()
+		}()
+
+		var dirCount, fileCount uint64
+		var recvErr error
+		for {
+			resp, err := stream.Recv()
+			if err != nil {
+				if !errors.Is(err, io.EOF) {
+					recvErr = fmt.Errorf("traverse %s: %w", path, err)
+				}
+				break
+			}
+			if resp.Entry == nil {
+				continue
+			}
+			protoMessage := &filer_pb.FullEntry{
+				Dir:   resp.Directory,
+				Entry: resp.Entry,
+			}
+			if genErr := genFn(ctx, protoMessage, outputChan); genErr != nil {
+				recvErr = genErr
+				cancel()
+				break
+			}
+			if resp.Entry.IsDirectory {
+				dirCount++
+			} else {
+				fileCount++
+			}
+			if verbose {
+				println(util.NewFullPath(resp.Directory, resp.Entry.Name))
+			}
+		}
+
+		close(outputChan)
+		wg.Wait()
+		saveErr := <-saveErrChan
+
+		if recvErr != nil {
+			return recvErr
+		}
+		if saveErr != nil {
+			return saveErr
+		}
+		if writer != nil {
+			fmt.Fprintf(writer, "collected filer entries from a consistent store snapshot\n")
+			fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
+		}
+		return nil
+	})
+	return used, err
+}
+
 func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error {
 
 	ctx, cancel := context.WithCancel(context.Background())

weed/shell/command_volume_fsck.go

@@ -63,6 +63,7 @@ type commandVolumeFsck struct {
 	tempFolder                string
 	verbose                   *bool
 	forcePurging              *bool
+	purgeConcurrency          *int
 	skipEcVolumes             *bool
 	findMissingChunksInFiler  *bool
 	verifyNeedle              *bool
@@ -99,6 +100,12 @@ func (c *commandVolumeFsck) Help() string {
 		A newly uploaded chunk may appear as orphan if metadata commit is still pending.
 		The default 5h cutoff provides sufficient buffer for metadata commits.
 
+	Step 2 reads the filer from one consistent snapshot when its store supports
+	that (the SQL stores do), so a file renamed while the filer is being read is
+	still counted and its chunks are not mistaken for orphans. Other stores are
+	walked directory by directory; there, avoid running with -reallyDeleteFromVolume
+	while files are being renamed or moved.
+
 `
 }
 
@@ -117,6 +124,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
 	applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer. Currently this only works with default filerGroup.")
 	c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging")
 	purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, "<expert only!> delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler")
+	c.purgeConcurrency = fsckCommand.Int("purgeConcurrency", 16, "parallel filer deletes for -reallyDeleteFilerEntries")
 	tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files")
 	cutoffTimeAgo := fsckCommand.Duration("cutoffTimeAgo", 5*time.Hour, "only include entries on volume servers before this cutoff time to check orphan chunks")
 	modifyTimeAgo := fsckCommand.Duration("modifyTimeAgo", 0, "only include entries after this modify time to check orphan chunks")
@@ -148,7 +156,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
 		}
 	}
 	c.env = commandEnv
-	c.writer = writer
+	c.writer = &lockedWriter{w: writer}
 
 	if c.collectionMatcher, err = wildcard.CompileCollectionMatcher(*c.collection); err != nil {
 		return fmt.Errorf("invalid collection pattern %q: %w", *c.collection, err)
@@ -322,7 +330,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 		}
 	}()
 
-	return doTraverseBfsAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, false,
+	return doTraverseAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, false,
 		func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
 			if *c.verbose && entry.Entry.IsDirectory {
 				fmt.Fprintf(c.writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))
@@ -360,10 +368,11 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 				}
 				select {
 				case outputChan <- &Item{
-					vid:     chunk.Fid.VolumeId,
-					fileKey: chunk.Fid.FileKey,
-					cookie:  chunk.Fid.Cookie,
-					path:    util.NewFullPath(entry.Dir, entry.Entry.Name),
+					vid:        chunk.Fid.VolumeId,
+					fileKey:    chunk.Fid.FileKey,
+					cookie:     chunk.Fid.Cookie,
+					path:       util.NewFullPath(entry.Dir, entry.Entry.Name),
+					hardLinkId: entry.Entry.GetHardLinkId(),
 				}:
 				case <-ctx.Done():
 					return ctx.Err()
@@ -373,6 +382,12 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 		},
 		func(outputChan chan interface{}) error {
 			buffer := make([]byte, readbufferSize)
+			var queuePurge func(purgeItem)
+			var finishPurge func()
+			if purgeAbsent {
+				queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete)
+				defer finishPurge()
+			}
 			for item := range outputChan {
 				i := item.(*Item)
 				if f, ok := files[i.vid]; ok {
@@ -400,7 +415,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m
 					fmt.Fprintf(c.writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
 					if purgeAbsent {
 						fmt.Fprintf(c.writer, "deleting path %s after volume not found\n", i.path)
-						c.httpDelete(i.path)
+						queuePurge(purgeItem{path: i.path, hardLinkId: i.hardLinkId})
 					}
 				}
 			}
@@ -615,10 +630,24 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
 }
 
 type Item struct {
-	vid     uint32
-	fileKey uint64
-	cookie  uint32
-	path    util.FullPath
+	vid        uint32
+	fileKey    uint64
+	cookie     uint32
+	path       util.FullPath
+	hardLinkId []byte
+}
+
+// lockedWriter serializes concurrent writers — the traversal, the consumer,
+// and the purge workers all log through c.writer at the same time.
+type lockedWriter struct {
+	mu sync.Mutex
+	w  io.Writer
+}
+
+func (lw *lockedWriter) Write(p []byte) (int, error) {
+	lw.mu.Lock()
+	defer lw.mu.Unlock()
+	return lw.w.Write(p)
 }
 
 func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath)) error {
@@ -686,6 +715,47 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, vo
 	return nil
 }
 
+type purgeItem struct {
+	path       util.FullPath
+	hardLinkId []byte
+}
+
+// startPurgeWorkers fans deletes across at least one worker and returns a
+// queue function plus a finish function that closes the channels and waits
+// for in-flight deletes. Paths that share a hard link id always land on the
+// same worker, so their shared hard-link counter updates stay serialized —
+// concurrent deletes of the same id could both read count N and write N-1,
+// leaving the shared record behind after every path is gone.
+func startPurgeWorkers(workers int, deleteFn func(util.FullPath)) (queue func(purgeItem), finish func()) {
+	if workers < 1 {
+		workers = 1
+	}
+	chans := make([]chan purgeItem, workers)
+	var wg sync.WaitGroup
+	for w := range chans {
+		chans[w] = make(chan purgeItem, 128)
+		wg.Add(1)
+		go func(ch <-chan purgeItem) {
+			defer wg.Done()
+			for it := range ch {
+				deleteFn(it.path)
+			}
+		}(chans[w])
+	}
+	return func(it purgeItem) {
+			key := []byte(it.path)
+			if len(it.hardLinkId) > 0 {
+				key = it.hardLinkId
+			}
+			chans[int(uint32(util.HashToInt32(key))%uint32(workers))] <- it
+		}, func() {
+			for _, ch := range chans {
+				close(ch)
+			}
+			wg.Wait()
+		}
+}
+
 func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
 	req, err := http.NewRequest(http.MethodDelete, "", nil)
 	if err != nil {

weed/shell/command_volume_fsck_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http/httptest"
 	"sort"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -226,3 +227,86 @@ func TestVolumeFsckPurgeEmptyDirectoriesKeepsChangedDirectory(t *testing.T) {
 		t.Errorf("deleted %v, expected a directory written to after the lookup to be kept", stub.deleted)
 	}
 }
+
+// The purge pool may run at most `workers` deletes at once, and at least one
+// even when the flag is set to zero.
+func TestVolumeFsckPurgeWorkerLimit(t *testing.T) {
+	gate := make(chan struct{})
+	var inFlight, maxSeen, processed atomic.Int32
+	queue, finish := startPurgeWorkers(3, func(util.FullPath) {
+		n := inFlight.Add(1)
+		for {
+			m := maxSeen.Load()
+			if n <= m || maxSeen.CompareAndSwap(m, n) {
+				break
+			}
+		}
+		<-gate
+		inFlight.Add(-1)
+		processed.Add(1)
+	})
+
+	for i := 0; i < 10; i++ {
+		queue(purgeItem{path: util.FullPath(fmt.Sprintf("/p/%d", i))})
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for inFlight.Load() < 3 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	if got := inFlight.Load(); got != 3 {
+		t.Fatalf("in-flight deletes %d, want the 3-worker limit", got)
+	}
+	time.Sleep(10 * time.Millisecond)
+	if got := maxSeen.Load(); got > 3 {
+		t.Fatalf("saw %d concurrent deletes, want at most 3", got)
+	}
+	close(gate)
+	finish()
+	if got := processed.Load(); got != 10 {
+		t.Fatalf("processed %d deletes, want all 10 drained", got)
+	}
+}
+
+func TestVolumeFsckPurgeWorkersAtLeastOne(t *testing.T) {
+	var processed atomic.Int32
+	queue, finish := startPurgeWorkers(0, func(util.FullPath) { processed.Add(1) })
+	for i := 0; i < 5; i++ {
+		queue(purgeItem{path: util.FullPath(fmt.Sprintf("/p/%d", i))})
+	}
+	finish()
+	if got := processed.Load(); got != 5 {
+		t.Fatalf("processed %d deletes, want 5", got)
+	}
+}
+
+// Deletes of paths sharing a hard link id must never overlap: the filer
+// decrements a shared counter non-atomically, so concurrent deletes of the
+// same id would both read N and write back N-1.
+func TestVolumeFsckPurgeHardLinksSerialized(t *testing.T) {
+	var inFlight, maxSeen, processed atomic.Int32
+	gate := make(chan struct{})
+	queue, finish := startPurgeWorkers(8, func(util.FullPath) {
+		n := inFlight.Add(1)
+		for {
+			m := maxSeen.Load()
+			if n <= m || maxSeen.CompareAndSwap(m, n) {
+				break
+			}
+		}
+		<-gate
+		inFlight.Add(-1)
+		processed.Add(1)
+	})
+	for i := 0; i < 8; i++ {
+		queue(purgeItem{path: util.FullPath(fmt.Sprintf("/hl/%d", i)), hardLinkId: []byte{9, 9, 9}})
+	}
+	time.Sleep(50 * time.Millisecond)
+	if got := maxSeen.Load(); got > 1 {
+		t.Fatalf("saw %d concurrent deletes on one hard link id, want 1", got)
+	}
+	close(gate)
+	finish()
+	if got := processed.Load(); got != 8 {
+		t.Fatalf("processed %d deletes, want 8", got)
+	}
+}

weed/shell/commands.go

@@ -26,12 +26,13 @@ type ShellOptions struct {
 	Masters        *string
 	GrpcDialOption grpc.DialOption
 	// shell transient context
-	FilerHost    string
-	FilerPort    int64
-	FilerGroup   *string
-	FilerAddress pb.ServerAddress
-	Directory    string
-	Debug        bool
+	FilerHost      string
+	FilerPort      int64
+	FilerGroup     *string
+	FilerAddress   pb.ServerAddress
+	Directory      string
+	Debug          bool
+	ConnectTimeout time.Duration
 }
 
 type CommandEnv struct {

weed/shell/shell_liner.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -57,14 +58,28 @@ func RunShell(options ShellOptions) error {
 
 	commandEnv := NewCommandEnv(&options)
 
+	connectTimeout := options.ConnectTimeout
+	if connectTimeout <= 0 {
+		connectTimeout = 30 * time.Second
+	}
 	ctx := context.Background()
-	go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
-	commandEnv.MasterClient.WaitUntilConnected(ctx)
+	connectCtx, cancel := context.WithTimeout(ctx, connectTimeout)
+	defer cancel()
+	reconnectCtx, cancelReconnect := context.WithCancel(ctx)
+	defer cancelReconnect()
+	go commandEnv.MasterClient.KeepConnectedToMaster(reconnectCtx)
+	if commandEnv.MasterClient.GetMaster(connectCtx) == "" {
+		err := fmt.Errorf("cannot connect to master %s within %s", *options.Masters, connectTimeout)
+		fmt.Fprintln(os.Stderr, err)
+		return err
+	}
 
 	if commandEnv.option.FilerAddress == "" {
 		var filers []pb.ServerAddress
-		commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
-			resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+		discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout)
+		defer cancelDiscover()
+		listErr := commandEnv.MasterClient.WithClient(discoverCtx, false, func(client master_pb.SeaweedClient) error {
+			resp, err := client.ListClusterNodes(discoverCtx, &master_pb.ListClusterNodesRequest{
 				ClientType: cluster.FilerType,
 				FilerGroup: *options.FilerGroup,
 			})
@@ -79,6 +94,8 @@ func RunShell(options ShellOptions) error {
 		})
 		if len(filers) > 0 {
 			commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))]
+		} else if listErr != nil {
+			fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr)
 		}
 		if options.Debug {
 			if len(filers) > 0 {

weed/shell/shell_liner_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"strings"
 	"testing"
+	"time"
 )
 
 type fakeCommand struct {
@@ -44,6 +45,24 @@ func TestProcessEachCmdReturnsErrors(t *testing.T) {
 	}
 }
 
+func TestRunShellConnectTimeout(t *testing.T) {
+	masters := "127.0.0.1:1"
+	filerGroup := ""
+
+	start := time.Now()
+	err := RunShell(ShellOptions{
+		Masters:        &masters,
+		FilerGroup:     &filerGroup,
+		ConnectTimeout: 200 * time.Millisecond,
+	})
+	if err == nil || !strings.Contains(err.Error(), "cannot connect to master") {
+		t.Fatalf("expected a master connection timeout, got: %v", err)
+	}
+	if elapsed := time.Since(start); elapsed > 10*time.Second {
+		t.Fatalf("RunShell should give up near the connect timeout, took %v", elapsed)
+	}
+}
+
 func TestProcessEachCmdPreservesRegisteredCommandError(t *testing.T) {
 	doErr := errors.New("daily_run: shard=3: recovery walk failed")
 	failing := &fakeCommand{name: "test.failing.command", err: doErr}