seaweedfs/weed/operation/volume_move/ec_move.go

package volume_move

import (
	"context"
	"fmt"
	"io"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)

// EcShardMove is one planned relocation of mounted shards to another server.
type EcShardMove struct {
	VolumeId   needle.VolumeId
	Collection string
	ShardIds   []erasure_coding.ShardId
	Source     pb.ServerAddress
	Target     pb.ServerAddress
	// TargetDisk picks the destination disk (0 lets the server pick).
	TargetDisk uint32
}

// EcMoveOptions control MoveEcShards.
type EcMoveOptions struct {
	// IoBytePerSecond limits the shard copy rate; 0 falls back to the volume
	// server's maintenance rate.
	IoBytePerSecond int64
	// Writer receives human-readable progress lines (nil discards them).
	Writer io.Writer
	// Progress, when set, receives percent/stage callbacks as the move advances.
	Progress func(percent float64, stage string)
}

// MoveEcShards relocates mounted shards: copy+mount on the target, verify the
// target registered them, then unmount+delete on the source. The verification
// gates the destructive half — a copy/mount RPC can return OK while the shard
// is not loadable on the target, and deleting the source then would lose it;
// on a mismatch the source is kept so the caller can retry.
func (m *Mover) MoveEcShards(ctx context.Context, move EcShardMove, opts EcMoveOptions) error {
	writer := opts.Writer
	if writer == nil {
		writer = io.Discard
	}
	progress := opts.Progress
	if progress == nil {
		progress = func(float64, string) {}
	}

	// A same-server "move" cannot be expressed with these RPCs: the source
	// delete is server-wide, so it would erase the just-copied shard. Removing
	// a duplicate shard in place is RemoveEcShards. SameServer, not ==:
	// "node:8080" and "node:8080.18080" are one server.
	if SameServer(move.Source, move.Target) {
		return fmt.Errorf("refusing EC shard move of volume %d shard(s) %v onto its own server %s: the source delete is server-wide", move.VolumeId, move.ShardIds, move.Source)
	}

	progress(10, fmt.Sprintf("copying EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
	if err := m.CopyAndMountEcShards(ctx, move.VolumeId, move.Collection, move.ShardIds, move.Source, move.Target, move.TargetDisk, opts.IoBytePerSecond, writer); err != nil {
		return err
	}

	progress(40, fmt.Sprintf("verifying EC shard(s) %d.%v on %s", move.VolumeId, move.ShardIds, move.Target))
	if err := m.VerifyEcShards(ctx, move.VolumeId, move.Target, move.ShardIds); err != nil {
		return err
	}

	progress(50, fmt.Sprintf("unmounting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
	fmt.Fprintf(writer, "unmount %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
	if err := m.UnmountEcShards(ctx, move.VolumeId, move.Source, move.ShardIds); err != nil {
		return fmt.Errorf("unmount %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
	}

	progress(75, fmt.Sprintf("deleting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
	fmt.Fprintf(writer, "delete %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
	if err := m.DeleteEcShards(ctx, move.VolumeId, move.Collection, move.Source, move.ShardIds); err != nil {
		return fmt.Errorf("delete %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
	}

	progress(100, fmt.Sprintf("moved EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
	return nil
}

// CopyAndMountEcShards has the target copy the shards (with their .ecx/.ecj/
// .vif/.ecsum sidecars) from source and mount them. A same-address call skips
// the copy and just mounts — ec.encode uses that to bring freshly generated
// shards online in place.
func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, source, target pb.ServerAddress, targetDisk uint32, ioBytePerSecond int64, writer io.Writer) error {
	if writer == nil {
		writer = io.Discard
	}
	// The target dials the embedded source itself; a malformed one would
	// abort the target server, not this client.
	if !SameServer(target, source) {
		if err := checkDialable(source); err != nil {
			return err
		}
	}
	return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
		if !SameServer(target, source) {
			fmt.Fprintf(writer, "copy %d.%v %s => %s\n", volumeId, shardIds, source, target)
			_, copyErr := client.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
				VolumeId:        uint32(volumeId),
				Collection:      collection,
				ShardIds:        erasure_coding.ShardIdsToUint32(shardIds),
				CopyEcxFile:     true,
				CopyEcjFile:     true,
				CopyVifFile:     true,
				CopyEcsumFile:   true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
				SourceDataNode:  string(source),
				DiskId:          targetDisk,
				IoBytePerSecond: ioBytePerSecond,
			})
			if copyErr != nil {
				return fmt.Errorf("copy %d.%v %s => %s: %v", volumeId, shardIds, source, target, copyErr)
			}
		}
		fmt.Fprintf(writer, "mount %d.%v on %s\n", volumeId, shardIds, target)
		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
		})
		if mountErr != nil {
			return fmt.Errorf("mount %d.%v on %s: %v", volumeId, shardIds, target, mountErr)
		}
		return nil
	})
}

// VerifyEcShards confirms server has every shard in shardIds registered for
// the volume.
func (m *Mover) VerifyEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
		resp, err := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
			VolumeId: uint32(volumeId),
		})
		if err != nil {
			return fmt.Errorf("verify EC shard(s) on %s for volume %d: %v", server, volumeId, err)
		}
		var bits, zeroSized erasure_coding.ShardBits
		for _, s := range resp.EcShardInfos {
			if s.VolumeId != uint32(volumeId) || s.ShardId >= erasure_coding.MaxShardCount {
				continue
			}
			// A zero-sized shard is residue of a failed operation, not a
			// shard; counting it as present would let a broken copy pass
			// verification and the source be deleted behind it.
			if s.Size <= 0 {
				zeroSized = zeroSized.Set(erasure_coding.ShardId(s.ShardId))
				continue
			}
			bits = bits.Set(erasure_coding.ShardId(s.ShardId))
		}
		for _, sid := range shardIds {
			if bits.Has(sid) {
				continue
			}
			if zeroSized.Has(sid) {
				return fmt.Errorf("%s has a zero-sized EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
			}
			return fmt.Errorf("%s missing EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
		}
		return nil
	})
}

// MountEcShards mounts shards already on server.
func (m *Mover) MountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
		})
		return mountErr
	})
}

// UnmountEcShards unmounts shards on server.
func (m *Mover) UnmountEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
		_, unmountErr := client.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
			VolumeId: uint32(volumeId),
			ShardIds: erasure_coding.ShardIdsToUint32(shardIds),
		})
		return unmountErr
	})
}

// DeleteEcShards deletes shards on server. The delete is server-wide: it
// removes the shards from every disk of the server.
func (m *Mover) DeleteEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
		_, deleteErr := client.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
			VolumeId:   uint32(volumeId),
			Collection: collection,
			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
		})
		return deleteErr
	})
}

// IsNoSpaceLeftError matches the volume server's refusal when no local disk
// can take the copied shard ("no space left" from FindEcShardTargetLocation,
// an ENOSPC writing to a caller-pinned disk, and the Rust server's same
// reply), however the move chain wraps it.
func IsNoSpaceLeftError(err error) bool {
	return err != nil && strings.Contains(err.Error(), "no space left")
}

// RemoveEcShards unmounts then deletes shards in place — the dedup path for a
// shard that already has a copy elsewhere.
func (m *Mover) RemoveEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
	if err := m.UnmountEcShards(ctx, volumeId, server, shardIds); err != nil {
		return fmt.Errorf("unmount %d.%v from %s: %v", volumeId, shardIds, server, err)
	}
	if err := m.DeleteEcShards(ctx, volumeId, collection, server, shardIds); err != nil {
		return fmt.Errorf("delete %d.%v from %s: %v", volumeId, shardIds, server, err)
	}
	return nil
}