mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
* operation: add shared volume_move package for volume and EC shard moves The shell commands (volume.move, volume.balance, ec.balance, tier moves) and the maintenance workers (balance, ec_balance) each carried their own copy of the move RPC sequences, and the copies had drifted: the worker verified the target before deleting the source but dropped the disk type and IO throttle; the shell passed those but deleted the source unverified. volume_move.Mover carries the merged sequences, keeping the stricter behavior from each side: - LiveMoveVolume: check-then-hard-freeze the source (VolumeStatus's IsReadOnly also covers low-disk and readonly-but-can-delete states, which still accept needle deletes), copy with disk type and IO throttle, tail, verify the target is not behind the source before the destructive source delete (a target that is ahead holds writes it accepted during the tail and the move commits to keep them), and restore the source's writability when a failure precedes the delete and this move did the freezing. Aborts clean up the incomplete target copy; a failed cleanup or an ambiguous source delete keeps the source readonly (ErrSourceKeptReadonly) so callers do not thaw a source next to a possibly-authoritative copy. With a readonly source, an existing or unknown-state target refuses the move outright: no client-side observation can prove such a copy is a stale remnant rather than the authoritative copy of an unfinished move. - MoveEcShards: copy with the .ecx/.ecj/.vif/.ecsum sidecars, mount, verify the target registered every shard before unmount+delete on the source, and reject same-server moves (the EC delete is server-wide). Server identity is the grpc endpoint (SameServer), so node:8080 and node:8080.18080 compare equal while test servers sharing a degenerate HTTP address stay distinct; addresses are validated non-fatally before dialing and before being embedded in copy/tail requests, since both the client dialer and the receiving server normalize them through a parser that aborts the process on a malformed port. The Rust volume server's codes.NotFound counts as a definitively absent probe answer alongside the Go server's plain-error code Unknown. All RPCs go through an injectable ClientFunc, so the sequences are unit tested against a fake volume server client: RPC order, request fields, and that verification failures keep the source intact. * shell, worker: delegate volume and EC shard moves to operation/volume_move LiveMoveVolume and the copy/tail/delete/mark-writable helpers become thin wrappers over the shared mover, keeping their signatures; the EC helpers keep their per-step output and delegate the RPCs. BalanceTask and ECBalanceTask keep their parameter validation, progress reporting, and guards (same-node cross-disk rejection, dedup keep-node verification, shard ids range-checked before the uint8 narrowing) and hand the RPC sequences to the mover. volume.tier.move skips its thaw-on-failure when the mover deliberately kept the source readonly, since reopening the replicas beside a possibly-authoritative target copy would fork the volume. The tail-failure tolerance moves inside the mover: a failed tail is tolerated only when the volume was already readonly before the move began, backstopped by a stability re-read across the idle window, so volume.balance's -skipTailError-by-readonly heuristic and tier-move's unconditional skip both become the same authoritative rule. * volume_move: keep the source readonly when a failed copy leaves a target of unknown origin A failed copy can leave a complete, mounted copy on the target (the server finishes after the client loses the stream). The abort probed the target only when its pre-copy state was known-absent; an unknown prior state skipped both the probe and the cleanup and then reopened the source - two writable replicas of one volume, diverging from the next write on. The abort now probes the target on every failed copy and restores the source only when the target provably holds nothing. A copy whose provenance cannot be proven (unknown prior state, a pre-existing replica, or an unreachable target) is never deleted, and the source stays readonly with ErrSourceKeptReadonly naming the recovery. * test: teach the plugin worker harness the shared move sequence The fake volume server lacked VolumeStatus, which the shared mover now issues before freezing the source, and the batch execution test's status-read accounting predates the pre-copy target probe and the verification reads. Mirrors the harness the enterprise tree already carries.
135 lines
3.9 KiB
Go
135 lines
3.9 KiB
Go
package balance
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
// BalanceTask implements the Task interface
|
|
type BalanceTask struct {
|
|
*base.BaseTask
|
|
server string
|
|
volumeID uint32
|
|
collection string
|
|
progress float64
|
|
grpcDialOption grpc.DialOption
|
|
}
|
|
|
|
// NewBalanceTask creates a new balance task instance
|
|
func NewBalanceTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *BalanceTask {
|
|
return &BalanceTask{
|
|
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
|
|
server: server,
|
|
volumeID: volumeID,
|
|
collection: collection,
|
|
grpcDialOption: grpcDialOption,
|
|
}
|
|
}
|
|
|
|
// Execute implements the Task interface
|
|
func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
|
|
if params == nil {
|
|
return fmt.Errorf("task parameters are required")
|
|
}
|
|
|
|
balanceParams := params.GetBalanceParams()
|
|
if balanceParams == nil {
|
|
return fmt.Errorf("balance parameters are required")
|
|
}
|
|
|
|
// Get source and destination from unified arrays
|
|
if len(params.Sources) == 0 {
|
|
return fmt.Errorf("source is required for balance task")
|
|
}
|
|
if len(params.Targets) == 0 {
|
|
return fmt.Errorf("target is required for balance task")
|
|
}
|
|
|
|
sourceNode := params.Sources[0].Node
|
|
destNode := params.Targets[0].Node
|
|
|
|
if sourceNode == "" {
|
|
return fmt.Errorf("source node is required for balance task")
|
|
}
|
|
if destNode == "" {
|
|
return fmt.Errorf("destination node is required for balance task")
|
|
}
|
|
|
|
t.GetLogger().WithFields(map[string]interface{}{
|
|
"volume_id": t.volumeID,
|
|
"source": sourceNode,
|
|
"destination": destNode,
|
|
"collection": t.collection,
|
|
}).Info("Starting balance task - moving volume")
|
|
|
|
// The move sequence — freeze the source, copy, tail, verify the target
|
|
// matches the source before the destructive delete — is shared with the
|
|
// shell's volume.move/volume.balance commands.
|
|
mover := volume_move.NewMover(t.grpcDialOption)
|
|
err := mover.LiveMoveVolume(ctx, needle.VolumeId(t.volumeID), pb.ServerAddress(sourceNode), pb.ServerAddress(destNode), volume_move.VolumeMoveOptions{
|
|
IdleTimeout: 60 * time.Second,
|
|
Progress: func(percent float64, stage string) {
|
|
t.ReportProgress(percent)
|
|
t.GetLogger().Info(stage)
|
|
},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("move volume %d from %s to %s: %w", t.volumeID, sourceNode, destNode, err)
|
|
}
|
|
|
|
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s",
|
|
t.volumeID, sourceNode, destNode)
|
|
return nil
|
|
}
|
|
|
|
// Validate implements the UnifiedTask interface
|
|
func (t *BalanceTask) Validate(params *worker_pb.TaskParams) error {
|
|
if params == nil {
|
|
return fmt.Errorf("task parameters are required")
|
|
}
|
|
|
|
balanceParams := params.GetBalanceParams()
|
|
if balanceParams == nil {
|
|
return fmt.Errorf("balance parameters are required")
|
|
}
|
|
|
|
if params.VolumeId != t.volumeID {
|
|
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
|
|
}
|
|
|
|
// Validate that at least one source matches our server
|
|
found := false
|
|
for _, source := range params.Sources {
|
|
if source.Node == t.server {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("no source matches expected server %s", t.server)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// EstimateTime implements the UnifiedTask interface
|
|
func (t *BalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
|
|
// Basic estimate based on simulated steps
|
|
return 14 * time.Second // Sum of all step durations
|
|
}
|
|
|
|
// GetProgress returns current progress
|
|
func (t *BalanceTask) GetProgress() float64 {
|
|
return t.progress
|
|
}
|