seaweedfs/weed/pb/filer_pb/filer_client_bfs_test.go

package filer_pb

import (
	"context"
	"fmt"
	"net"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

// stubFiler serves canned listings and counts how often each directory is listed.
type stubFiler struct {
	UnimplementedSeaweedFilerServer
	mu       sync.Mutex
	listed   map[string]int
	listings func(req *ListEntriesRequest, send func(*Entry) error) error
}

func (s *stubFiler) ListEntries(req *ListEntriesRequest, stream SeaweedFiler_ListEntriesServer) error {
	s.mu.Lock()
	if s.listed == nil {
		s.listed = make(map[string]int)
	}
	s.listed[req.Directory]++
	s.mu.Unlock()
	return s.listings(req, func(entry *Entry) error {
		return stream.Send(&ListEntriesResponse{Entry: entry})
	})
}

func (s *stubFiler) listedCount(dir string) int {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.listed[dir]
}

type stubFilerClient struct {
	conn *grpc.ClientConn
}

func (c *stubFilerClient) WithFilerClient(streamingMode bool, fn func(SeaweedFilerClient) error) error {
	return fn(NewSeaweedFilerClient(c.conn))
}

func (c *stubFilerClient) AdjustedUrl(location *Location) string { return location.Url }

func (c *stubFilerClient) GetDataCenter() string { return "" }

func startStubFiler(t *testing.T, s *stubFiler) *stubFilerClient {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	grpcServer := grpc.NewServer()
	RegisterSeaweedFilerServer(grpcServer, s)
	go grpcServer.Serve(listener)
	t.Cleanup(grpcServer.Stop)
	conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { conn.Close() })
	return &stubFilerClient{conn: conn}
}

// A directory delivered twice by a misbehaving listing must still be walked once.
func TestTraverseBfsVisitsDuplicateDirectoryOnce(t *testing.T) {
	server := &stubFiler{}
	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
		switch req.Directory {
		case "/":
			if req.StartFromFileName != "" {
				return nil
			}
			for _, name := range []string{"dup", "dup"} {
				if err := send(&Entry{Name: name, IsDirectory: true}); err != nil {
					return err
				}
			}
		case "/dup":
			if req.StartFromFileName != "" {
				return nil
			}
			for _, name := range []string{"a.txt", "b.txt"} {
				if err := send(&Entry{Name: name}); err != nil {
					return err
				}
			}
		}
		return nil
	}
	filerClient := startStubFiler(t, server)

	var mu sync.Mutex
	seen := make(map[string]int)
	err := TraverseBfs(context.Background(), filerClient, "/", func(parentPath util.FullPath, entry *Entry) error {
		mu.Lock()
		seen[string(parentPath.Child(entry.Name))]++
		mu.Unlock()
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}

	if got := server.listedCount("/dup"); got != 1 {
		t.Fatalf("listed /dup %d times, want 1", got)
	}
	for _, path := range []string{"/dup/a.txt", "/dup/b.txt"} {
		if seen[path] != 1 {
			t.Fatalf("visited %s %d times, want 1", path, seen[path])
		}
	}
}

// A store whose pagination never advances past the cursor must error out
// instead of re-listing the same page forever.
func TestReadDirAllEntriesStuckPagination(t *testing.T) {
	server := &stubFiler{}
	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
		for i := uint32(0); i < req.Limit; i++ {
			if err := send(&Entry{Name: fmt.Sprintf("f%05d", i)}); err != nil {
				return err
			}
		}
		return nil
	}
	filerClient := startStubFiler(t, server)

	done := make(chan error, 1)
	go func() {
		done <- ReadDirAllEntries(context.Background(), filerClient, "/", "", func(entry *Entry, isLast bool) error {
			return nil
		})
	}()

	select {
	case err := <-done:
		if err == nil || !strings.Contains(err.Error(), "pagination stuck") {
			t.Fatalf("want pagination stuck error, got %v", err)
		}
	case <-time.After(30 * time.Second):
		t.Fatal("listing loops forever on a non-advancing store")
	}
}

// A start path with a trailing slash must still descend into subdirectories.
func TestTraverseBfsTrailingSlashRoot(t *testing.T) {
	server := &stubFiler{}
	server.listings = func(req *ListEntriesRequest, send func(*Entry) error) error {
		if req.StartFromFileName != "" {
			return nil
		}
		// the filer trims a single trailing slash, nothing more
		switch strings.TrimSuffix(req.Directory, "/") {
		case "/buckets/data":
			return send(&Entry{Name: "sub", IsDirectory: true})
		case "/buckets/data/sub":
			return send(&Entry{Name: "a.txt"})
		}
		return nil
	}
	filerClient := startStubFiler(t, server)

	var mu sync.Mutex
	var seen []string
	err := TraverseBfs(context.Background(), filerClient, "/buckets/data/", func(parentPath util.FullPath, entry *Entry) error {
		mu.Lock()
		seen = append(seen, fmt.Sprintf("%s -> %s", parentPath, entry.Name))
		mu.Unlock()
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}

	sort.Strings(seen)
	want := []string{"/buckets/data -> sub", "/buckets/data/sub -> a.txt"}
	if !reflect.DeepEqual(seen, want) {
		t.Fatalf("visited %v, want %v", seen, want)
	}
}