seaweedfs/weed/filer/filer_delete_collection_test.go

package filer

import (
	"context"
	"fmt"
	"net"
	"os"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"

	"github.com/seaweedfs/seaweedfs/weed/cluster"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
	"github.com/seaweedfs/seaweedfs/weed/wdclient"
)

// collectionDeleteMaster is just enough of a master for a MasterClient to
// consider itself connected, and records every CollectionDelete it is asked for.
type collectionDeleteMaster struct {
	master_pb.UnimplementedSeaweedServer
	calls chan collectionDeleteCall
}

type collectionDeleteCall struct {
	name string
	// budget is the time the RPC arrived with, or 0 when it carried no deadline.
	budget time.Duration
}

// KeepConnected is what MasterClient waits on before it reports a master: one
// response is enough, then the stream idles until the server is torn down.
func (m *collectionDeleteMaster) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {
	if _, err := stream.Recv(); err != nil {
		return err
	}
	if err := stream.Send(&master_pb.KeepConnectedResponse{}); err != nil {
		return err
	}
	<-stream.Context().Done()
	return nil
}

func (m *collectionDeleteMaster) CollectionDelete(ctx context.Context, req *master_pb.CollectionDeleteRequest) (*master_pb.CollectionDeleteResponse, error) {
	call := collectionDeleteCall{name: req.Name}
	if deadline, ok := ctx.Deadline(); ok {
		call.budget = time.Until(deadline)
	}
	m.calls <- call
	return &master_pb.CollectionDeleteResponse{}, nil
}

// hookedStore is the stub store with one extra seam: a callback that runs once
// an entry has actually been removed, so a test can interleave an event -- a
// client hanging up, say -- between the store write and whatever follows it.
type hookedStore struct {
	*stubFilerStore
	onDeleteEntry func(util.FullPath)
}

func (s *hookedStore) DeleteEntry(ctx context.Context, p util.FullPath) error {
	if err := s.stubFilerStore.DeleteEntry(ctx, p); err != nil {
		return err
	}
	if s.onDeleteEntry != nil {
		s.onDeleteEntry(p)
	}
	return nil
}

// newFilerWithFakeMaster builds a filer backed by the stub store, with its
// MasterClient connected to a fake master the test can observe.
func newFilerWithFakeMaster(t *testing.T) (*Filer, *hookedStore, *collectionDeleteMaster) {
	t.Helper()

	master := &collectionDeleteMaster{calls: make(chan collectionDeleteCall, 4)}

	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	master_pb.RegisterSeaweedServer(grpcServer, master)
	go func() { _ = grpcServer.Serve(lis) }()
	t.Cleanup(grpcServer.Stop)

	_, port, err := net.SplitHostPort(lis.Addr().String())
	if err != nil {
		t.Fatalf("split listener address: %v", err)
	}
	masterAddress := pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))

	mc := wdclient.NewMasterClient(
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		"test", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
		*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": masterAddress}),
	)

	connecting, stopConnecting := context.WithCancel(context.Background())
	t.Cleanup(stopConnecting)
	go mc.KeepConnectedToMaster(connecting)

	waiting, cancelWaiting := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancelWaiting()
	mc.WaitUntilConnected(waiting)
	if waiting.Err() != nil {
		t.Fatal("the master client never connected to the fake master")
	}

	store := &hookedStore{stubFilerStore: newStubFilerStore()}
	f := &Filer{
		DirBucketsPath:      "/buckets",
		RemoteStorage:       NewFilerRemoteStorage(),
		Store:               NewFilerStoreWrapper(store),
		FilerConf:           NewFilerConf(),
		MaxFilenameLength:   255,
		MasterClient:        mc,
		FileIdDeletionQueue: util.NewUnboundedQueue(),
		deletionQuit:        make(chan struct{}),
		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
	}
	return f, store, master
}

// Deleting a bucket entry deletes its collection, and the request hanging up
// partway must not stop that: the bucket's metadata is already gone, so skipping
// it strands the bucket's volumes with nothing left to come back for them. The
// cleanup still has to carry a deadline of its own, or a master that is down
// parks this handler and every client retry behind it parks another.
//
// The cancellation lands between the store removing the entry and the collection
// delete, which is where a client disconnect actually bites: FilerStoreWrapper
// refuses a context that is already dead on entry, so an up-front cancellation
// fails the delete long before this point instead.
func TestDeleteEntryMetaAndDataDeletesCollectionWhenTheRequestIsCancelledMidDelete(t *testing.T) {
	f, store, master := newFilerWithFakeMaster(t)

	const bucket = "bucket-a"
	bucketPath := util.FullPath(f.DirBucketsPath + "/" + bucket)
	if err := store.InsertEntry(context.Background(), &Entry{
		FullPath: bucketPath,
		Attr:     Attr{Mode: os.ModeDir | 0755},
	}); err != nil {
		t.Fatalf("seed the bucket entry: %v", err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	// The client hangs up just as the bucket entry comes out of the store.
	store.onDeleteEntry = func(util.FullPath) { cancel() }

	if err := f.DeleteEntryMetaAndData(ctx, bucketPath, true, false, true, false, nil, 0); err != nil {
		t.Fatalf("DeleteEntryMetaAndData: %v", err)
	}
	if ctx.Err() == nil {
		t.Fatal("test setup: the request was never cancelled, so nothing was exercised")
	}

	var call collectionDeleteCall
	select {
	case call = <-master.calls:
	case <-time.After(20 * time.Second):
		t.Fatal("CollectionDelete never reached the master")
	}
	if call.name != bucket {
		t.Errorf("master was asked to delete collection %q, want %q", call.name, bucket)
	}
	if call.budget <= 0 {
		t.Error("CollectionDelete arrived with no deadline; a master that stops answering would hold this handler open")
	}
	if call.budget > collectionDeleteTimeout {
		t.Errorf("CollectionDelete budget = %v, want at most collectionDeleteTimeout %v", call.budget, collectionDeleteTimeout)
	}

	if store.getEntry(string(bucketPath)) != nil {
		t.Error("the bucket entry survived the delete")
	}
}