filer.meta.backup: resume an interrupted metadata copy and add -entriesPerSecond (#12549)

* filer_pb: add a depth-first traversal that can resume after a path

TraverseDfs walks a tree in listing order and reports each finished
path, so a caller can save one path and continue from it later. Pages
are read whole before visiting, and the entries of a page are visited
by up to five workers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.meta.backup: resume an interrupted metadata copy

The walk saves the last finished path every few seconds, and a walk
that continues an unfinished one starts after it. A fresh walk clears
the saved path first, and a finished walk clears it with the walk
start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.meta.backup: add -entriesPerSecond to throttle the metadata copy

Each entry the full copy reads waits on a rate limiter, and pages are
listed only after the previous page is copied, so the flag also bounds
how fast the copy lists the source filer. 0 keeps it unlimited.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer_pb: report each file of a depth-first walk once all before it are done

A throttled walk only reported progress after a whole listing page, so a
slow copy could go many minutes without a resumable path. Visit entries in
runs that end at a directory and report each file as soon as every entry
before it is done.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

weed/pb/filer_pb/filer_client_dfs.go

@@ -0,0 +1,124 @@
+package filer_pb
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var traverseDfsPageSize uint32 = 1024
+
+// TraverseDfs walks the tree under parentPath depth first in listing order, so
+// one path tells how far it got: every entry before it was visited, and so was
+// the subtree of a directory it names. progress gets these paths in walk
+// order, and a walk given one as resumeAfter skips everything up to it. fn may
+// run concurrently for the files of one directory.
+func TraverseDfs(ctx context.Context, filerClient FilerClient, parentPath, resumeAfter util.FullPath, fn func(parentPath util.FullPath, entry *Entry) error, progress func(walked util.FullPath)) error {
+	parentPath = util.NormalizePath(string(parentPath))
+	var resume []string
+	if resumeAfter != "" && resumeAfter != parentPath && resumeAfter.IsUnder(parentPath) {
+		glog.V(0).Infof("traverse %s after %s", parentPath, resumeAfter)
+		resume = util.FullPath(strings.TrimPrefix(string(resumeAfter), strings.TrimSuffix(string(parentPath), "/"))).Split()
+	}
+	return traverseDfsDirectory(ctx, filerClient, parentPath, resume, fn, progress)
+}
+
+func traverseDfsDirectory(ctx context.Context, filerClient FilerClient, dir util.FullPath, resume []string, fn func(parentPath util.FullPath, entry *Entry) error, progress func(walked util.FullPath)) error {
+	startFrom, inclusive := "", false
+	if len(resume) > 0 {
+		startFrom, inclusive = resume[0], true
+	}
+	for {
+		var page []*Entry
+		if err := List(ctx, filerClient, string(dir), "", func(entry *Entry, isLast bool) error {
+			page = append(page, entry)
+			return nil
+		}, startFrom, inclusive, traverseDfsPageSize); err != nil {
+			return err
+		}
+
+		listed := page
+		// the entry the walk stopped at: done with its subtree when it ends the
+		// path, else a directory to resume inside
+		var resumeIn []string
+		if inclusive && len(page) > 0 && page[0].Name == startFrom {
+			if len(resume) == 1 {
+				page = page[1:]
+			} else if page[0].IsDirectory {
+				resumeIn = resume[1:]
+			}
+		}
+
+		if resumeIn != nil {
+			child := dir.Child(page[0].Name)
+			if err := traverseDfsDirectory(ctx, filerClient, child, resumeIn, fn, progress); err != nil {
+				return err
+			}
+			progress(child)
+			page = page[1:]
+		}
+		for len(page) > 0 {
+			run := page
+			if i := slices.IndexFunc(page, func(entry *Entry) bool { return entry.IsDirectory }); i >= 0 {
+				run = page[:i+1]
+			}
+			if err := visitDfsRun(ctx, dir, run, fn, progress); err != nil {
+				return err
+			}
+			if last := run[len(run)-1]; last.IsDirectory {
+				child := dir.Child(last.Name)
+				if err := traverseDfsDirectory(ctx, filerClient, child, nil, fn, progress); err != nil {
+					return err
+				}
+				progress(child)
+			}
+			page = page[len(run):]
+		}
+
+		if uint32(len(listed)) < traverseDfsPageSize {
+			return nil
+		}
+		last := listed[len(listed)-1].Name
+		if !inclusive && last == startFrom {
+			return fmt.Errorf("list %s: pagination stuck at %q", dir, startFrom)
+		}
+		startFrom, inclusive = last, false
+	}
+}
+
+// visitDfsRun calls fn concurrently on entries that end at the first
+// directory, and reports each file once every entry before it is done. The
+// caller reports the directory after its subtree.
+func visitDfsRun(ctx context.Context, dir util.FullPath, run []*Entry, fn func(parentPath util.FullPath, entry *Entry) error, progress func(walked util.FullPath)) error {
+	var mu sync.Mutex
+	done := make([]bool, len(run))
+	next := 0
+	g, gctx := errgroup.WithContext(ctx)
+	g.SetLimit(5)
+	for i, entry := range run {
+		g.Go(func() error {
+			if err := gctx.Err(); err != nil {
+				return err
+			}
+			if err := fn(dir, entry); err != nil {
+				return err
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			done[i] = true
+			for next < len(run) && done[next] && !run[next].IsDirectory {
+				progress(dir.Child(run[next].Name))
+				next++
+			}
+			return nil
+		})
+	}
+	return g.Wait()
+}

weed/pb/filer_pb/filer_client_dfs_test.go

@@ -0,0 +1,159 @@
+package filer_pb
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func startTreeFiler(t *testing.T, tree map[string][]string) *stubFilerClient {
+	t.Helper()
+	server := &stubFiler{}
+	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
+		names := append([]string(nil), tree[req.Directory]...)
+		sort.Strings(names)
+		sent := uint32(0)
+		for _, name := range names {
+			if req.StartFromFileName != "" && (name < req.StartFromFileName || name == req.StartFromFileName && !req.InclusiveStartFrom) {
+				continue
+			}
+			if sent == req.Limit {
+				break
+			}
+			_, isDir := tree[string(util.FullPath(req.Directory).Child(name))]
+			if err := send(&Entry{Name: name, IsDirectory: isDir}); err != nil {
+				return err
+			}
+			sent++
+		}
+		return nil
+	}
+	return startStubFiler(t, server)
+}
+
+func traverseDfsForTest(t *testing.T, filerClient FilerClient, root, resumeAfter util.FullPath) (visited map[string]int, progressed []string) {
+	t.Helper()
+	var mu sync.Mutex
+	visited = make(map[string]int)
+	err := TraverseDfs(context.Background(), filerClient, root, resumeAfter, func(parentPath util.FullPath, entry *Entry) error {
+		mu.Lock()
+		visited[string(parentPath.Child(entry.Name))]++
+		mu.Unlock()
+		return nil
+	}, func(walked util.FullPath) {
+		progressed = append(progressed, string(walked))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return visited, progressed
+}
+
+func TestTraverseDfsResumesAfterEachProgressPath(t *testing.T) {
+	saved := traverseDfsPageSize
+	traverseDfsPageSize = 2
+	t.Cleanup(func() { traverseDfsPageSize = saved })
+
+	filerClient := startTreeFiler(t, map[string][]string{
+		"/":    {"a", "b", "c", "d", "e"},
+		"/a":   {"x", "y", "z"},
+		"/a/y": {"1", "2", "3"},
+		"/c":   {},
+		"/e":   {"f"},
+	})
+
+	visited, progressed := traverseDfsForTest(t, filerClient, "/", "")
+	want := []string{"/a/x", "/a/y/1", "/a/y/2", "/a/y/3", "/a/y", "/a/z", "/a", "/b", "/c", "/d", "/e/f", "/e"}
+	if !reflect.DeepEqual(progressed, want) {
+		t.Fatalf("progress %v, want %v", progressed, want)
+	}
+	if len(visited) != len(want) {
+		t.Fatalf("visited %v, want each of %v", visited, want)
+	}
+	for _, p := range want {
+		if visited[p] != 1 {
+			t.Fatalf("visited %s %d times, want once", p, visited[p])
+		}
+	}
+
+	for i, resumeAfter := range want {
+		visited, progressed := traverseDfsForTest(t, filerClient, "/", util.FullPath(resumeAfter))
+		if !slices.Equal(progressed, want[i+1:]) {
+			t.Fatalf("after %s: progress %v, want %v", resumeAfter, progressed, want[i+1:])
+		}
+		for _, p := range want[i+1:] {
+			wantVisits := 1
+			if strings.HasPrefix(resumeAfter, p+"/") {
+				wantVisits = 0
+			}
+			if visited[p] != wantVisits {
+				t.Fatalf("after %s: visited %s %d times, want %d", resumeAfter, p, visited[p], wantVisits)
+			}
+			delete(visited, p)
+		}
+		if len(visited) != 0 {
+			t.Fatalf("after %s: visited %v again", resumeAfter, visited)
+		}
+	}
+}
+
+func TestTraverseDfsResumeAfterAMissingOrForeignPath(t *testing.T) {
+	filerClient := startTreeFiler(t, map[string][]string{
+		"/":    {"a", "b"},
+		"/a":   {"x", "y"},
+		"/a/y": {"1", "2", "3"},
+	})
+
+	_, progressed := traverseDfsForTest(t, filerClient, "/a/", "/a/y/15")
+	if want := []string{"/a/y/2", "/a/y/3", "/a/y"}; !reflect.DeepEqual(progressed, want) {
+		t.Fatalf("after a deleted path: progress %v, want %v", progressed, want)
+	}
+
+	_, progressed = traverseDfsForTest(t, filerClient, "/a", "/b")
+	if want := []string{"/a/x", "/a/y/1", "/a/y/2", "/a/y/3", "/a/y"}; !reflect.DeepEqual(progressed, want) {
+		t.Fatalf("after a path outside the walk: progress %v, want %v", progressed, want)
+	}
+}
+
+func TestTraverseDfsReportsFilesBeforeTheirPageFinishes(t *testing.T) {
+	filerClient := startTreeFiler(t, map[string][]string{
+		"/":  {"a", "b", "c", "d"},
+		"/d": {"x"},
+	})
+
+	var mu sync.Mutex
+	var progressed []string
+	reachedB := make(chan struct{})
+	err := TraverseDfs(context.Background(), filerClient, "/", "", func(parentPath util.FullPath, entry *Entry) error {
+		if entry.Name != "c" {
+			return nil
+		}
+		select {
+		case <-reachedB:
+			return nil
+		case <-time.After(5 * time.Second):
+			return fmt.Errorf("no progress while %s was in flight", entry.Name)
+		}
+	}, func(walked util.FullPath) {
+		mu.Lock()
+		defer mu.Unlock()
+		progressed = append(progressed, string(walked))
+		if walked == "/b" {
+			close(reachedB)
+		}
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := []string{"/a", "/b", "/c", "/d/x", "/d"}; !reflect.DeepEqual(progressed, want) {
+		t.Fatalf("progress %v, want %v", progressed, want)
+	}
+}