seaweedfs/weed/server/block_rebuild_remote.go

package weed_server

import (
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)

// errRebuildAckFailed is returned by TransferFullBase when the rebuild failed
// via a replica SessionAckFailed. The ack observation path (ObserveReplicaRebuildSessionAck)
// already emitted engine.SessionFailed, so callers must NOT emit a second one.
var errRebuildAckFailed = errors.New("rebuild failed via session ack")

// RemoteRebuildIO implements engine.RebuildIO for the primary-side remote
// rebuild path. Instead of installing base blocks locally (which is what the
// v2bridge.Executor does on the replica), this implementation coordinates
// remotely: it sends a session control message to the replica and monitors
// acks until the replica reports completion or failure.
//
// The base blocks are served by the primary's existing RebuildServer.
// The replica auto-starts a base lane client on receiving the session control.
// WAL entries continue flowing through the shipper's data channel.
//
// Ack forwarding: each ack from the replica is forwarded to onAck, which
// routes through ObserveReplicaRebuildSessionAck for pin/watchdog/engine
// integration.
type RemoteRebuildIO struct {
	// ReplicaCtrlAddr is the replica's control channel address.
	ReplicaCtrlAddr string
	// RebuildAddr is the primary's rebuild server address (sent to replica
	// so it knows where to connect for the base lane).
	RebuildAddr string
	// BaseLSN is the flushed/checkpoint boundary the extent can serve.
	// Must be derived from the real checkpoint, not the plan target.
	BaseLSN uint64
	// Epoch is the current volume epoch for session validation.
	Epoch uint64
	// SessionID uniquely identifies this rebuild session.
	SessionID uint64
	// OnAck forwards each replica ack to the observation layer
	// (ObserveReplicaRebuildSessionAck). Returns error if the ack is
	// rejected (stale session, etc.). Shipper state transitions happen
	// only after successful observation.
	OnAck func(blockvol.SessionAckMsg) error
	// TransitionShipper changes the shipper state. Called only after
	// OnAck succeeds. Used for: NeedsRebuild → Rebuilding (on accepted),
	// Rebuilding → InSync (on completed), Rebuilding → NeedsRebuild (on failure).
	TransitionShipper func(blockvol.ReplicaState)
}

// TransferFullBase sends a session control to the replica and blocks until
// the rebuild completes or fails. committedLSN is the engine's frozen target
// from PlanRebuild — the replica must reach at least this LSN for completion.
//
// Protocol:
//  1. Dial replica ctrl addr (fresh connection, separate from barrier)
//  2. Send atomic SessionControlV2 with RebuildAddr trailer
//  3. Read acks in loop; forward each to OnAck for observation
//  4. On SessionAckAccepted: transition shipper to Rebuilding (live WAL lane opens)
//  5. On SessionAckCompleted: transition shipper to InSync, return achievedLSN
//  6. On SessionAckFailed or error: transition shipper to NeedsRebuild, return error
func (r *RemoteRebuildIO) TransferFullBase(committedLSN uint64) (uint64, error) {
	conn, err := net.DialTimeout("tcp", r.ReplicaCtrlAddr, 5*time.Second)
	if err != nil {
		return 0, fmt.Errorf("remote rebuild: dial ctrl %s: %w", r.ReplicaCtrlAddr, err)
	}
	defer conn.Close()

	// Set a generous deadline for the entire rebuild session.
	conn.SetDeadline(time.Now().Add(10 * time.Minute))

	// Send atomic v2 session control with RebuildAddr trailer.
	msg := blockvol.SessionControlMsg{
		Epoch:       r.Epoch,
		SessionID:   r.SessionID,
		Command:     blockvol.SessionCmdStartRebuild,
		BaseLSN:     r.BaseLSN,
		TargetLSN:   committedLSN, // engine's frozen rebuild target
		RebuildAddr: r.RebuildAddr,
	}
	if err := blockvol.SendSessionControlV2(conn, msg); err != nil {
		return 0, fmt.Errorf("remote rebuild: send session control: %w", err)
	}

	glog.V(0).Infof("remote rebuild: sent start_rebuild session=%d base=%d target=%d rebuild=%s → %s",
		r.SessionID, r.BaseLSN, committedLSN, r.RebuildAddr, r.ReplicaCtrlAddr)

	// Read acks until terminal phase.
	for {
		msgType, payload, err := blockvol.ReadFrame(conn)
		if err != nil {
			r.transitionOnFailure()
			return 0, fmt.Errorf("remote rebuild: read ack: %w", err)
		}
		if msgType != blockvol.MsgSessionAck {
			continue
		}
		ack, err := blockvol.DecodeSessionAck(payload)
		if err != nil {
			r.transitionOnFailure()
			return 0, fmt.Errorf("remote rebuild: decode ack: %w", err)
		}

		// Forward to observation layer. Only transition shipper state
		// if observation succeeds (Rule 1: ack-gated transitions).
		ackErr := r.forwardAck(ack)

		switch ack.Phase {
		case blockvol.SessionAckAccepted:
			if ackErr != nil {
				r.transitionOnFailure()
				return 0, fmt.Errorf("remote rebuild: accepted ack rejected by observation: %w", ackErr)
			}
			if r.TransitionShipper != nil {
				r.TransitionShipper(blockvol.ReplicaRebuilding)
			}
			glog.V(0).Infof("remote rebuild: session %d accepted by replica", r.SessionID)

		case blockvol.SessionAckRunning, blockvol.SessionAckBaseComplete:
			// Fail closed: if observation rejects a progress ack, the primary
			// no longer considers this session valid (stale ID, wrong kind, etc.).
			if ackErr != nil {
				r.transitionOnFailure()
				return 0, fmt.Errorf("remote rebuild: progress ack rejected by observation: %w", ackErr)
			}

		case blockvol.SessionAckCompleted:
			if ackErr != nil {
				r.transitionOnFailure()
				return 0, fmt.Errorf("remote rebuild: completed ack rejected by observation: %w", ackErr)
			}
			if r.TransitionShipper != nil {
				r.TransitionShipper(blockvol.ReplicaInSync)
			}
			achieved := ack.AchievedLSN
			if achieved == 0 {
				achieved = ack.WALAppliedLSN
			}
			glog.V(0).Infof("remote rebuild: session %d completed (achieved=%d)", r.SessionID, achieved)
			return achieved, nil

		case blockvol.SessionAckFailed:
			r.transitionOnFailure()
			if ackErr != nil {
				// Observation rejected the ack (stale session, etc.) — don't use
				// sentinel. Return a regular error so ExecutePendingRebuild emits
				// the fallback SessionFailed since observation didn't handle it.
				return 0, fmt.Errorf("remote rebuild: session %d failed (observation rejected: %w)", r.SessionID, ackErr)
			}
			// Observation accepted the ack and already emitted SessionFailed.
			// Use sentinel so ExecutePendingRebuild doesn't double-emit.
			return 0, fmt.Errorf("remote rebuild: session %d: %w", r.SessionID, errRebuildAckFailed)
		}
	}
}

// TransferSnapshot is not supported in the V1 remote rebuild path.
func (r *RemoteRebuildIO) TransferSnapshot(snapshotLSN uint64) error {
	return fmt.Errorf("remote rebuild: TransferSnapshot not supported (v1 full-base only)")
}

// StreamWALEntries is not supported — WAL flows through the shipper's data channel.
func (r *RemoteRebuildIO) StreamWALEntries(startExclusive, endInclusive uint64) (uint64, error) {
	return 0, fmt.Errorf("remote rebuild: StreamWALEntries not supported (WAL flows through shipper)")
}

func (r *RemoteRebuildIO) forwardAck(ack blockvol.SessionAckMsg) error {
	if r.OnAck == nil {
		return nil
	}
	return r.OnAck(ack)
}

func (r *RemoteRebuildIO) transitionOnFailure() {
	if r.TransitionShipper != nil {
		r.TransitionShipper(blockvol.ReplicaNeedsRebuild)
	}
}