seaweedfs/weed/pb/filer_pb/filer_client_bfs.go

package filer_pb

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"

	"github.com/seaweedfs/seaweedfs/weed/util"
)

func TraverseBfs(ctx context.Context, filerClient FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {
	K := 5

	// callers hand in user-typed paths, and every entry is reported relative
	// to this one, so a trailing slash must not reach the callback
	parentPath = util.NormalizePath(string(parentPath))

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	queue := util.NewQueue[util.FullPath]()
	var pending sync.WaitGroup
	pending.Add(1)
	queue.Enqueue(parentPath)

	var once sync.Once
	var firstErr error

	// A directory delivered twice (a page-boundary race with concurrent
	// renames, or a store whose listing order misbehaves) must not be walked
	// twice: the second walk re-lists the same subtree and can keep the
	// traversal from ever terminating.
	var visited sync.Map
	visited.Store(string(parentPath), struct{}{})

	enqueue := func(p util.FullPath) bool {
		// Stop expanding traversal once canceled (e.g. first error encountered).
		if ctx.Err() != nil {
			return false
		}
		if _, seen := visited.LoadOrStore(string(p), struct{}{}); seen {
			return true
		}
		pending.Add(1)
		queue.Enqueue(p)
		return true
	}

	done := make(chan struct{})
	var workers sync.WaitGroup
	for i := 0; i < K; i++ {
		workers.Add(1)
		go func() {
			defer workers.Done()
			for {
				select {
				case <-done:
					return
				default:
				}

				dir := queue.Dequeue()
				if dir == "" {
					// queue is empty for now
					select {
					case <-done:
						return
					case <-time.After(50 * time.Millisecond):
						continue
					}
				}

				// Always mark the directory as done so the closer can finish.
				if ctx.Err() == nil {
					processErr := processOneDirectory(ctx, filerClient, dir, enqueue, fn)
					if processErr != nil {
						once.Do(func() {
							firstErr = processErr
							cancel()
						})
					}
				}
				pending.Done()
			}
		}()
	}

	pending.Wait()
	close(done)

	workers.Wait()

	return firstErr
}

func processOneDirectory(ctx context.Context, filerClient FilerClient, parentPath util.FullPath, enqueue func(p util.FullPath) bool, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {

	return ReadDirAllEntries(ctx, filerClient, parentPath, "", func(entry *Entry, isLast bool) error {

		if err := fn(parentPath, entry); err != nil {
			return err
		}

		if entry.IsDirectory {
			if !enqueue(parentPath.Child(entry.Name)) {
				return ctx.Err()
			}
		}
		return nil
	})

}

func StreamBfs(ctx context.Context, client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry, entryTsNs int64) error) (err error) {
	glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs))
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{
		Directory: string(dir),
	})
	if err != nil {
		return fmt.Errorf("traverse bfs metadata: %w", err)
	}
	for {
		resp, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("traverse bfs metadata: %w", err)
		}
		if err := fn(util.FullPath(resp.Directory), resp.Entry, resp.EntryTsNs); err != nil {
			return err
		}
	}
	return nil
}