seaweedfs/weed/ec/ec_move_runner_test.go

package ec

import (
	"errors"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
)

// evacuationMoves is what volumeServer.evacuate plans: every shard of each
// volume leaves the same source node.
func evacuationMoves(volumes, shards int) []ecbalancer.Move {
	var moves []ecbalancer.Move
	for v := 0; v < volumes; v++ {
		for s := 0; s < shards; s++ {
			moves = append(moves, ecbalancer.Move{
				VolumeID:   uint32(100 + v),
				ShardID:    s,
				SourceNode: "source",
				TargetNode: []string{"node-a", "node-b", "node-c"}[(v+s)%3],
			})
		}
	}
	return moves
}

func TestRunEcShardMovesStopsOnFirstError(t *testing.T) {
	moves := evacuationMoves(3, 2)
	var ran []ecbalancer.Move
	result, err := runEcShardMoves(moves, 1, true, func(m ecbalancer.Move) error {
		ran = append(ran, m)
		if len(ran) == 2 {
			return errors.New("input/output error")
		}
		return nil
	})
	if err == nil {
		t.Fatal("expected the failed move's error")
	}
	if len(ran) != 2 {
		t.Fatalf("expected no move to start after the failure, %d ran", len(ran))
	}
	if result.Moved != 1 || result.Failed != 1 || result.NotStarted != len(moves)-2 {
		t.Fatalf("unexpected result %+v", result)
	}
}

func TestRunEcShardMovesGoesOnAfterFailure(t *testing.T) {
	moves := evacuationMoves(3, 3)
	broken := moves[0]
	ran := map[uint32]int{}
	result, err := runEcShardMoves(moves, 1, false, func(m ecbalancer.Move) error {
		ran[m.VolumeID]++
		if m.VolumeID == broken.VolumeID && m.ShardID == broken.ShardID {
			return errors.New("input/output error")
		}
		return nil
	})
	if err == nil {
		t.Fatal("expected the failed move's error")
	}
	if ran[broken.VolumeID] != 3 {
		t.Fatalf("expected the other shards of volume %d to move after its failed shard, %d moves ran", broken.VolumeID, ran[broken.VolumeID])
	}
	if result.Moved != len(moves)-1 || result.Failed != 1 || result.NotStarted != 0 {
		t.Fatalf("unexpected result %+v", result)
	}
	if len(result.FailedMoves) != 1 || result.FailedMoves[0] != broken {
		t.Fatalf("expected the broken move to be reported, got %+v", result.FailedMoves)
	}
}

func TestRunEcShardMovesInParallel(t *testing.T) {
	const parallelization = 4
	moves := evacuationMoves(20, 4)
	started := make(chan struct{}, len(moves))
	release := make(chan struct{})
	var mu sync.Mutex
	running, most := 0, 0
	var result EcShardMovesResult
	var err error
	done := make(chan struct{})
	go func() {
		defer close(done)
		result, err = runEcShardMoves(moves, parallelization, true, func(m ecbalancer.Move) error {
			mu.Lock()
			running++
			most = max(most, running)
			mu.Unlock()
			started <- struct{}{}
			<-release
			mu.Lock()
			running--
			mu.Unlock()
			return nil
		})
	}()
	// The first moves are held until parallelization of them have started.
	for i := 0; i < parallelization; i++ {
		select {
		case <-started:
		case <-time.After(5 * time.Second):
			t.Fatalf("only %d moves started at once, expected %d", i, parallelization)
		}
	}
	close(release)
	<-done
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if result.Moved != len(moves) || result.Failed != 0 || result.NotStarted != 0 {
		t.Fatalf("unexpected result %+v", result)
	}
	if most != parallelization {
		t.Fatalf("%d moves ran at once, expected %d", most, parallelization)
	}
}

func TestRunEcShardMovesStopsWhenLockIsLost(t *testing.T) {
	moves := evacuationMoves(3, 2)
	ran := 0
	result, err := runEcShardMoves(moves, 1, false, func(m ecbalancer.Move) error {
		ran++
		if ran == 2 {
			return ErrLockLost
		}
		return nil
	})
	if !errors.Is(err, ErrLockLost) {
		t.Fatalf("expected the lost lock to be reported, got %v", err)
	}
	if ran != 2 || result.Moved != 1 || result.Failed != 1 || result.NotStarted != len(moves)-2 {
		t.Fatalf("expected no move after the lock was lost even with skipping on: ran %d, %+v", ran, result)
	}
}

// A move that failed after its target registered the shard counts as failed but
// is not reported for planning again: the target's slot is taken.
func TestRunEcShardMovesKeepsShardsOnTheTargetOutOfFailedMoves(t *testing.T) {
	moves := evacuationMoves(2, 1)
	result, err := runEcShardMoves(moves, 1, false, func(m ecbalancer.Move) error {
		if m.VolumeID == moves[0].VolumeID {
			return fmt.Errorf("move ec shard: %w: delete: input/output error", errShardOnTarget)
		}
		return errors.New("copy: input/output error")
	})
	if err == nil {
		t.Fatal("expected the failed moves' errors")
	}
	if result.Failed != 2 || len(result.FailedMoves) != 1 || result.FailedMoves[0] != moves[1] {
		t.Fatalf("expected only the move without a target copy in FailedMoves, got %+v", result)
	}
	if len(result.LandedMoves) != 1 || result.LandedMoves[0] != moves[0] {
		t.Fatalf("expected the move with a target copy in LandedMoves, got %+v", result)
	}
}