seaweedfs/weed/mount/weedfs_meta_subscribe_test.go

package mount

import (
	"bytes"
	"context"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"path/filepath"
	"sync/atomic"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/protobuf/proto"

	"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// proxyFilerServer replays the previous minute's metadata as a persisted log
// chunk, the way a filer answers every fresh subscription, and points volume
// lookups at an address nothing listens on: the mount host's view of a volume
// server it is meant to reach only through -volumeServerAccess=filerProxy.
type proxyFilerServer struct {
	filer_pb.UnimplementedSeaweedFilerServer
	chunk         *filer_pb.FileChunk
	subscriptions atomic.Int32
	volumeLookups atomic.Int32
	stop          chan struct{}
}

func (s *proxyFilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) error {
	if s.subscriptions.Add(1) > 1 {
		<-s.stop
		return nil
	}
	return stream.Send(&filer_pb.SubscribeMetadataResponse{
		LogFileRefs: []*filer_pb.LogFileChunkRef{{
			FilerId:  "5319c0e8",
			FileTsNs: time.Now().UnixNano(),
			Chunks:   []*filer_pb.FileChunk{s.chunk},
		}},
	})
}

func (s *proxyFilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
	s.volumeLookups.Add(1)
	locations := make(map[string]*filer_pb.Locations)
	for _, vid := range req.VolumeIds {
		locations[vid] = &filer_pb.Locations{Locations: []*filer_pb.Location{{Url: "127.0.0.1:1", PublicUrl: "127.0.0.1:1"}}}
	}
	return &filer_pb.LookupVolumeResponse{LocationsMap: locations}, nil
}

// buildLogFileData writes the on-disk log format: [4-byte size | LogEntry].
func buildLogFileData(events ...*filer_pb.SubscribeMetadataResponse) []byte {
	var buf bytes.Buffer
	for _, event := range events {
		eventData, _ := proto.Marshal(event)
		entryData, _ := proto.Marshal(&filer_pb.LogEntry{TsNs: event.TsNs, Data: eventData, Key: []byte(event.Directory)})
		sizeBuf := make([]byte, 4)
		util.Uint32toBytes(sizeBuf, uint32(len(entryData)))
		buf.Write(sizeBuf)
		buf.Write(entryData)
	}
	return buf.Bytes()
}

// TestSubscribeMetaEventsReplaysThroughFilerProxy pins the replay to the lookup
// the mount was given: resolving volume servers here fails every subscription
// of a filerProxy mount, which then resubscribes a second later, forever.
func TestSubscribeMetaEventsReplaysThroughFilerProxy(t *testing.T) {
	logData := buildLogFileData(&filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		TsNs:      time.Now().UnixNano(),
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 7}},
		},
	})

	var proxiedChunkIds atomic.Value
	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fileId := r.URL.Query().Get("proxyChunkId")
		if fileId == "" {
			http.Error(w, "not a filer proxy read", http.StatusBadRequest)
			return
		}
		proxiedChunkIds.Store(fileId)
		w.Write(logData)
	}))
	t.Cleanup(proxy.Close)

	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	t.Cleanup(func() { _ = listener.Close() })

	testServer := &proxyFilerServer{
		chunk: &filer_pb.FileChunk{FileId: "4471,ce598bfff8416e", Size: uint64(len(logData))},
		stop:  make(chan struct{}),
	}
	server := pb.NewGrpcServer()
	filer_pb.RegisterSeaweedFilerServer(server, testServer)
	go server.Serve(listener)
	t.Cleanup(func() {
		close(testServer.stop)
		server.Stop()
	})

	proxyUrl, err := url.Parse(proxy.URL)
	if err != nil {
		t.Fatalf("parse proxy url: %v", err)
	}

	uidGidMapper, err := meta_cache.NewUidGidMapper("", "")
	if err != nil {
		t.Fatalf("create uid/gid mapper: %v", err)
	}
	root := util.FullPath("/")
	wfs := &WFS{
		signature:         1,
		inodeToPath:       NewInodeToPath(root, 0),
		fhMap:             NewFileHandleToInode(),
		fhLockTable:       util.NewLockTable[FileHandleId](),
		hardLinkLockTable: util.NewLockTable[string](),
		option: &Option{
			ChunkSizeLimit:     1024,
			ConcurrentReaders:  1,
			VolumeServerAccess: "filerProxy",
			FilerMountRootPath: "/",
			FilerAddresses: []pb.ServerAddress{
				pb.NewServerAddressWithGrpcPort(proxyUrl.Host, listener.Addr().(*net.TCPAddr).Port),
			},
			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
		},
	}
	wfs.metaCache = meta_cache.NewMetaCache(
		filepath.Join(t.TempDir(), "meta"),
		uidGidMapper,
		root,
		false,
		func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) },
		func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) },
		func(meta_cache.EntryInvalidation) {},
		nil,
	)
	t.Cleanup(wfs.metaCache.Shutdown)
	wfs.inodeToPath.MarkChildrenCached(root)
	wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false)
	wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))

	followed := make(chan struct{})
	go func() {
		defer close(followed)
		meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.LookupFn(), wfs.option.FilerMountRootPath, 0, false, nil)
	}()

	select {
	case <-followed:
	case <-time.After(20 * time.Second):
		t.Fatal("the log chunk replay never succeeded")
	}

	if got := testServer.volumeLookups.Load(); got != 0 {
		t.Errorf("filerProxy mount resolved %d volume locations, want 0", got)
	}
	if got, _ := proxiedChunkIds.Load().(string); got != testServer.chunk.FileId {
		t.Errorf("filer proxy served chunk %q, want %q", got, testServer.chunk.FileId)
	}

	entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file"))
	if err != nil {
		t.Fatalf("replayed entry: %v", err)
	}
	if entry.Attr.FileSize != 7 {
		t.Errorf("replayed file size %d, want 7", entry.Attr.FileSize)
	}
}