seaweedfs/weed/shell/command_volume_tier_compact_test.go

package shell

import (
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/wdclient"
	"github.com/stretchr/testify/assert"
)

func buildTestTopologyWithRemoteVolumes() *master_pb.TopologyInfo {
	return &master_pb.TopologyInfo{
		Id: "topo",
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				Id: "dc1",
				RackInfos: []*master_pb.RackInfo{
					{
						Id: "rack1",
						DataNodeInfos: []*master_pb.DataNodeInfo{
							{
								Id:       "server1:8080",
								GrpcPort: 18080,
								DiskInfos: map[string]*master_pb.DiskInfo{
									"": {
										VolumeInfos: []*master_pb.VolumeInformationMessage{
											{
												Id:                1,
												Collection:        "col1",
												RemoteStorageName: "s3.default",
												RemoteStorageKey:  "col1/1.dat",
												DeleteCount:       100,
												DeletedByteCount:  1000,
												Size:              5000,
											},
											{
												Id:                2,
												Collection:        "col1",
												RemoteStorageName: "",
												RemoteStorageKey:  "",
												Size:              3000,
											},
											{
												Id:                3,
												Collection:        "col2",
												RemoteStorageName: "s3.archive",
												RemoteStorageKey:  "col2/3.dat",
												DeleteCount:       50,
												DeletedByteCount:  500,
												Size:              2000,
											},
										},
									},
								},
							},
						},
					},
				},
			},
			{
				Id: "dc2",
				RackInfos: []*master_pb.RackInfo{
					{
						Id: "rack1",
						DataNodeInfos: []*master_pb.DataNodeInfo{
							{
								Id:       "server2:8080",
								GrpcPort: 18080,
								DiskInfos: map[string]*master_pb.DiskInfo{
									"": {
										VolumeInfos: []*master_pb.VolumeInformationMessage{
											{
												Id:                4,
												Collection:        "bucket1",
												RemoteStorageName: "s3.default",
												RemoteStorageKey:  "bucket1/4.dat",
												Size:              8000,
											},
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}
}

func TestFindRemoteVolumeInTopology(t *testing.T) {
	topo := buildTestTopologyWithRemoteVolumes()

	tests := []struct {
		name       string
		vid        needle.VolumeId
		collection string
		wantFound  bool
		wantDest   string
	}{
		{
			name:      "find existing remote volume",
			vid:       1,
			wantFound: true,
			wantDest:  "s3.default",
		},
		{
			name:      "find remote volume with different backend",
			vid:       3,
			wantFound: true,
			wantDest:  "s3.archive",
		},
		{
			name:      "find remote volume on another server",
			vid:       4,
			wantFound: true,
			wantDest:  "s3.default",
		},
		{
			name:      "local volume not found as remote",
			vid:       2,
			wantFound: false,
		},
		{
			name:      "non-existent volume not found",
			vid:       999,
			wantFound: false,
		},
		{
			name:       "filter by matching collection exact",
			vid:        1,
			collection: "^col1$",
			wantFound:  true,
			wantDest:   "s3.default",
		},
		{
			name:       "filter by matching collection regex",
			vid:        1,
			collection: "col.*",
			wantFound:  true,
			wantDest:   "s3.default",
		},
		{
			name:       "filter by non-matching collection",
			vid:        1,
			collection: "^col2$",
			wantFound:  false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			rv, found, err := findRemoteVolumeInTopology(topo, tt.vid, tt.collection)
			assert.NoError(t, err)
			assert.Equal(t, tt.wantFound, found)
			if found {
				assert.Equal(t, tt.vid, rv.vid)
				assert.Equal(t, tt.wantDest, rv.remoteStorageName)
			}
		})
	}
}

func TestFindRemoteVolumeInTopologyInvalidPattern(t *testing.T) {
	topo := buildTestTopologyWithRemoteVolumes()

	_, _, err := findRemoteVolumeInTopology(topo, 1, "[invalid")
	assert.Error(t, err)
}

func TestCollectRemoteVolumesWithInfo(t *testing.T) {
	topo := buildTestTopologyWithRemoteVolumes()

	tests := []struct {
		name              string
		collectionPattern string
		wantCount         int
		wantVids          []needle.VolumeId
	}{
		{
			name:              "empty pattern matches empty collection only",
			collectionPattern: "",
			wantCount:         0,
		},
		{
			name:              "match all collections",
			collectionPattern: ".*",
			wantCount:         3, // volumes 1, 3, 4
			wantVids:          []needle.VolumeId{1, 3, 4},
		},
		{
			name:              "match specific collection",
			collectionPattern: "^col1$",
			wantCount:         1,
			wantVids:          []needle.VolumeId{1},
		},
		{
			name:              "match collection prefix",
			collectionPattern: "col.*",
			wantCount:         2, // volumes 1, 3
			wantVids:          []needle.VolumeId{1, 3},
		},
		{
			name:              "match bucket collection",
			collectionPattern: "bucket1",
			wantCount:         1,
			wantVids:          []needle.VolumeId{4},
		},
		{
			name:              "no match",
			collectionPattern: "^nonexistent$",
			wantCount:         0,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result, err := collectRemoteVolumesWithInfo(topo, tt.collectionPattern)
			assert.NoError(t, err)
			assert.Equal(t, tt.wantCount, len(result))

			if tt.wantVids != nil {
				gotVids := make(map[needle.VolumeId]bool)
				for _, rv := range result {
					gotVids[rv.vid] = true
				}
				for _, vid := range tt.wantVids {
					assert.True(t, gotVids[vid], "expected volume %d in results", vid)
				}
			}
		})
	}
}

func TestCollectRemoteVolumesWithInfoCaptures(t *testing.T) {
	topo := buildTestTopologyWithRemoteVolumes()

	result, err := collectRemoteVolumesWithInfo(topo, "^col1$")
	assert.NoError(t, err)
	assert.Equal(t, 1, len(result))

	rv := result[0]
	assert.Equal(t, needle.VolumeId(1), rv.vid)
	assert.Equal(t, "col1", rv.collection)
	assert.Equal(t, "s3.default", rv.remoteStorageName)
	assert.Equal(t, "server1:8080", rv.serverUrl)
	assert.NotEmpty(t, rv.serverAddress)
}

func TestCollectRemoteVolumesWithInfoInvalidPattern(t *testing.T) {
	topo := buildTestTopologyWithRemoteVolumes()

	_, err := collectRemoteVolumesWithInfo(topo, "[invalid")
	assert.Error(t, err)
}

func buildTestTopologyWithReplicas() *master_pb.TopologyInfo {
	mkDn := func(id string, volumes ...*master_pb.VolumeInformationMessage) *master_pb.DataNodeInfo {
		return &master_pb.DataNodeInfo{
			Id:       id,
			GrpcPort: 18080,
			DiskInfos: map[string]*master_pb.DiskInfo{
				"": {VolumeInfos: volumes},
			},
		}
	}
	remote := func(vid uint32) *master_pb.VolumeInformationMessage {
		return &master_pb.VolumeInformationMessage{
			Id:                vid,
			Collection:        "col1",
			RemoteStorageName: "s3.default",
			RemoteStorageKey:  "col1/vid.dat",
		}
	}
	local := func(vid uint32) *master_pb.VolumeInformationMessage {
		return &master_pb.VolumeInformationMessage{Id: vid, Collection: "col1"}
	}
	return &master_pb.TopologyInfo{
		DataCenterInfos: []*master_pb.DataCenterInfo{
			{
				RackInfos: []*master_pb.RackInfo{
					{
						DataNodeInfos: []*master_pb.DataNodeInfo{
							mkDn("n1:8080", remote(1)),
							mkDn("n2:8080", remote(1), local(2)),
							mkDn("n3:8080", remote(1)),
						},
					},
				},
			},
		},
	}
}

func TestCollectVolumeTierCompactLocations(t *testing.T) {
	topo := buildTestTopologyWithReplicas()

	tiered, local := collectVolumeTierCompactLocations(topo, needle.VolumeId(1), "col1")
	assert.Equal(t, 3, len(tiered))
	assert.Equal(t, 0, len(local))

	tiered, local = collectVolumeTierCompactLocations(topo, needle.VolumeId(2), "col1")
	assert.Equal(t, 0, len(tiered))
	assert.Equal(t, 1, len(local))

	// a different collection or volume id matches nothing
	tiered, local = collectVolumeTierCompactLocations(topo, needle.VolumeId(1), "other")
	assert.Equal(t, 0, len(tiered))
	assert.Equal(t, 0, len(local))
}

func TestPlanTierCompactResync(t *testing.T) {
	loc := func(url string) wdclient.Location {
		return wdclient.Location{Url: url, GrpcPort: 1}
	}
	urls := func(locs []wdclient.Location) (out []string) {
		for _, l := range locs {
			out = append(out, l.Url)
		}
		return
	}

	// single replica: nothing to resync, nothing to delete (its download
	// already removed the object)
	order, deleter := planTierCompactResync([]wdclient.Location{loc("a")}, nil)
	assert.Empty(t, order)
	assert.Empty(t, deleter)

	// two tiered replicas: the sibling is the last holder and goes last
	order, deleter = planTierCompactResync([]wdclient.Location{loc("a"), loc("b")}, nil)
	assert.Empty(t, order)
	assert.Equal(t, []string{"b"}, urls(deleter))

	// three tiered replicas plus one local copy: middle replicas resync in
	// order, the last tiered replica deletes and is resynced last
	order, deleter = planTierCompactResync(
		[]wdclient.Location{loc("a"), loc("b"), loc("c")},
		[]wdclient.Location{loc("d")})
	assert.Equal(t, []string{"b", "d"}, urls(order))
	assert.Equal(t, []string{"c"}, urls(deleter))
}