seaweedfs/weed/server/master_grpc_server_digest_test.go

package weed_server

import (
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/sequence"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/topology"
)

func digestTestCluster(t *testing.T) (*MasterServer, *topology.DataNode) {
	t.Helper()
	topo := topology.NewTopology("test", sequence.NewMemorySequencer(), 32*1024*1024*1024, 5, false)
	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
	return &MasterServer{Topo: topo}, dn
}

func digestTestVolumeMessage(id uint32) *master_pb.VolumeInformationMessage {
	return &master_pb.VolumeInformationMessage{
		Id: id, Size: 1024, Collection: "c", Version: 3,
	}
}

// A volume server that predates the digest keeps sending its whole list and
// must never be asked for anything, whatever the master computes. This is what
// lets the two sides be upgraded in either order.
func TestDigestCheckIgnoresServersThatReportNone(t *testing.T) {
	ms, dn := digestTestCluster(t)
	ms.Topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1)}, dn)

	if ms.checkVolumeDigest(&master_pb.Heartbeat{
		Volumes: []*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1)},
	}, dn) {
		t.Error("a server reporting no digest was asked to resend")
	}
}

func TestDigestCheckAcceptsAMatchingReport(t *testing.T) {
	ms, dn := digestTestCluster(t)
	volumes := []*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1), digestTestVolumeMessage(2)}
	ms.Topo.SyncDataNodeRegistration(volumes, dn)

	digest := dn.VolumeDigest()
	if ms.checkVolumeDigest(&master_pb.Heartbeat{Volumes: volumes, VolumeDigest: &digest}, dn) {
		t.Error("a matching digest was asked to resend")
	}
}

// A heartbeat that already carried the whole list has nothing further to give,
// so a mismatch there is a genuine disagreement to report rather than something
// to ask about again.
func TestDigestCheckDoesNotReaskAfterAFullList(t *testing.T) {
	ms, dn := digestTestCluster(t)
	volumes := []*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1)}
	ms.Topo.SyncDataNodeRegistration(volumes, dn)

	wrong := dn.VolumeDigest() ^ 1
	if ms.checkVolumeDigest(&master_pb.Heartbeat{Volumes: volumes, VolumeDigest: &wrong}, dn) {
		t.Error("a full volume list that still disagreed was asked to resend, which would repeat forever")
	}
}

// The case the request exists for: a heartbeat carrying no list whose digest
// disagrees means the master has drifted and needs the list back.
func TestDigestCheckAsksForTheListWhenADeltaDisagrees(t *testing.T) {
	ms, dn := digestTestCluster(t)
	ms.Topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1)}, dn)

	wrong := dn.VolumeDigest() ^ 1
	if !ms.checkVolumeDigest(&master_pb.Heartbeat{VolumeDigest: &wrong}, dn) {
		t.Error("a disagreeing digest with no list to fall back on was not asked to resend")
	}
}

// A node reporting one volume id twice is stored once, so the digests cannot
// agree however often the list is resent.
// A node reporting one volume id twice is stored once, so no digest can ever
// agree. It has to keep sending its whole list: nothing else would tell the
// master what it stopped holding.
func TestDigestCheckKeepsDuplicateNodesOnFullLists(t *testing.T) {
	ms, dn := digestTestCluster(t)
	duplicated := digestTestVolumeMessage(1)
	duplicated.DiskId = 1
	ms.Topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
		digestTestVolumeMessage(1), duplicated,
	}, dn)

	digest := dn.VolumeDigest()
	if !ms.checkVolumeDigest(&master_pb.Heartbeat{VolumeDigest: &digest}, dn) {
		t.Error("a node whose digest can never be verified was left sending only changes")
	}
	// And is not asked again for a list it just sent.
	if ms.checkVolumeDigest(&master_pb.Heartbeat{
		Volumes:      []*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1), duplicated},
		VolumeDigest: &digest,
	}, dn) {
		t.Error("a node that just sent its whole list was asked for it again")
	}
}

// The lookup index can drift from the disks without the volume server seeing
// anything, so its digest still matches. Only a full report re-registers the
// volumes that stopped being servable, and in delta mode nothing else asks for
// one.
func TestDigestCheckAsksForTheListWhenTheLookupIndexDrifts(t *testing.T) {
	ms, dn := digestTestCluster(t)
	volumes := []*master_pb.VolumeInformationMessage{digestTestVolumeMessage(1), digestTestVolumeMessage(2)}
	ms.Topo.SyncDataNodeRegistration(volumes, dn)

	digest := dn.VolumeDigest()
	if ms.checkVolumeDigest(&master_pb.Heartbeat{VolumeDigest: &digest}, dn) {
		t.Fatal("a healthy node was asked to resend")
	}

	rp, _ := super_block.NewReplicaPlacementFromString("000")
	vl := ms.Topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
	vl.SetVolumeUnavailable(dn, needle.VolumeId(1))

	if dn.VolumeDigest() != digest {
		t.Fatal("expected the reported digest to be unaffected, which is why the index has to be checked")
	}
	if !ms.checkVolumeDigest(&master_pb.Heartbeat{VolumeDigest: &digest}, dn) {
		t.Error("a volume that stopped being servable left the node sending only changes, so nothing would repair it")
	}
}

// A volume server takes the options from every response it receives, and
// preallocate is a bare bool with no way to tell "off" from "not mentioned". A
// response that left it out would turn preallocation off until reconnect.
func TestHeartbeatResponsesCarryTheVolumeOptions(t *testing.T) {
	ms := &MasterServer{
		option:          &MasterOption{VolumeSizeLimitMB: 1024},
		preallocateSize: 1,
		Topo:            topology.NewTopology("test", sequence.NewMemorySequencer(), 32*1024, 5, false),
	}

	resend := ms.heartbeatResponse()
	resend.ResendFullVolumeList = true

	if !resend.Preallocate {
		t.Error("a resend request would turn off preallocation on the volume server")
	}
	if resend.VolumeSizeLimit != 1024*1024*1024 {
		t.Errorf("a resend request carried volume size limit %d, want the configured one", resend.VolumeSizeLimit)
	}
}