seaweedfs/weed/ec/ec_move_runner.go

package ec

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"

	"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)

// ErrLockLost stops every move that has not started, whatever stopOnError says,
// and fails the whole run.
var ErrLockLost = errors.New("lock is lost")

// errShardOnTarget marks a failed move whose target registered the shard: only
// the source cleanup failed, so the shard takes a slot on the target.
var errShardOnTarget = errors.New("the target has the shard")

// EcShardMovesResult counts what RunEcShardMoves did with the planned moves.
type EcShardMovesResult struct {
	Moved      int
	Failed     int
	NotStarted int
	// FailedMoves are the failed moves whose target did not register the shard;
	// their shards are still on the source.
	FailedMoves []ecbalancer.Move
	// LandedMoves are the failed moves whose target has the shard.
	LandedMoves []ecbalancer.Move
}

// RunEcShardMoves applies planned shard moves, up to maxParallelization at once,
// scheduled like ec.balance moves (runMovesSpreadingTargets): never two moves of
// one volume at a time, and the free workers go to the least busy targets. Each
// move copies the shard, verifies the target registered it and only then deletes
// it on the source (volume_move.Mover.MoveEcShards). The in-memory model is not
// touched, so the caller updates it while planning.
//
// With stopOnError no new move starts after the first failure (moves already
// running finish). Otherwise every other move still runs, including the other
// shards of the failed move's volume: unlike ec.balance moves, evacuation moves
// all leave one server for other ones, so no move waits for a slot another frees.
// Losing the admin lock stops the moves that have not started in either case.
// The errors of the failed moves are joined.
func RunEcShardMoves(env *Env, nodes []*EcNode, moves []ecbalancer.Move, maxParallelization int, ioBytePerSecond int64, stopOnError bool, writer io.Writer) (EcShardMovesResult, error) {
	byID := make(map[string]*EcNode, len(nodes))
	for _, en := range nodes {
		byID[en.Info.Id] = en
	}
	out := &lockedWriter{w: writer}
	mover := volume_move.NewMover(env.GrpcDialOption)
	return runEcShardMoves(moves, maxParallelization, stopOnError, func(m ecbalancer.Move) error {
		if !env.isLocked() {
			return ErrLockLost
		}
		src, dst := byID[m.SourceNode], byID[m.TargetNode]
		if src == nil || dst == nil {
			return fmt.Errorf("move ec shard %d.%d %s => %s: unknown volume server", m.VolumeID, m.ShardID, m.SourceNode, m.TargetNode)
		}
		move := volume_move.EcShardMove{
			VolumeId:   needle.VolumeId(m.VolumeID),
			Collection: m.Collection,
			ShardIds:   []erasure_coding.ShardId{erasure_coding.ShardId(m.ShardID)},
			Source:     pb.NewServerAddressFromDataNode(src.Info),
			Target:     pb.NewServerAddressFromDataNode(dst.Info),
			TargetDisk: m.TargetDisk,
		}
		err := mover.MoveEcShards(context.Background(), move, volume_move.EcMoveOptions{IoBytePerSecond: ioBytePerSecond, Writer: out})
		if err != nil {
			if mover.VerifyEcShards(context.Background(), move.VolumeId, move.Target, move.ShardIds) == nil {
				err = fmt.Errorf("%w: %w", errShardOnTarget, err)
			}
			return fmt.Errorf("move ec shard %d.%d %s => %s: %w", m.VolumeID, m.ShardID, m.SourceNode, m.TargetNode, err)
		}
		fmt.Fprintf(out, "moved ec shard %d.%d %s => %s\n", m.VolumeID, m.ShardID, m.SourceNode, m.TargetNode)
		return nil
	})
}

// runEcShardMoves is RunEcShardMoves with the move itself passed in.
func runEcShardMoves(moves []ecbalancer.Move, maxParallelization int, stopOnError bool, move func(ecbalancer.Move) error) (EcShardMovesResult, error) {
	var stopped atomic.Bool
	var moved, failed atomic.Int64
	var mu sync.Mutex
	var errs []error
	var failedMoves, landedMoves []ecbalancer.Move
	err := runMovesSpreadingTargets(maxParallelization, moves, func(m ecbalancer.Move) error {
		if stopped.Load() {
			return nil
		}
		if err := move(m); err != nil {
			failed.Add(1)
			mu.Lock()
			if errors.Is(err, errShardOnTarget) {
				landedMoves = append(landedMoves, m)
			} else {
				failedMoves = append(failedMoves, m)
			}
			mu.Unlock()
			if stopOnError || errors.Is(err, ErrLockLost) {
				stopped.Store(true)
				return err
			}
			// Not returned, so the runner keeps the rest of this volume's moves.
			mu.Lock()
			errs = append(errs, err)
			mu.Unlock()
			return nil
		}
		moved.Add(1)
		return nil
	})
	result := EcShardMovesResult{
		Moved:       int(moved.Load()),
		Failed:      int(failed.Load()),
		FailedMoves: failedMoves,
		LandedMoves: landedMoves,
	}
	// Moves skipped after a stop, and moves dropped together with a failed move
	// of their volume, did not run.
	result.NotStarted = len(moves) - result.Moved - result.Failed
	return result, errors.Join(append(errs, err)...)
}

// lockedWriter serializes the progress lines of concurrent moves.
type lockedWriter struct {
	mu sync.Mutex
	w  io.Writer
}

func (lw *lockedWriter) Write(p []byte) (int, error) {
	lw.mu.Lock()
	defer lw.mu.Unlock()
	return lw.w.Write(p)
}