seaweedfs/weed/storage/blockvol/replica_apply.go

package blockvol

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"
)

var (
	ErrStaleEpoch   = errors.New("blockvol: stale epoch")
	ErrDuplicateLSN = errors.New("blockvol: duplicate or out-of-order LSN")
)

// ReplicaReceiver listens for WAL entries from a primary and applies them
// to the local BlockVol. It runs two listeners: one for the data channel
// (WAL entries) and one for the control channel (barrier requests).
type ReplicaReceiver struct {
	vol            *BlockVol
	barrierTimeout time.Duration
	advertisedHost string // canonical IP for address reporting; empty = auto-detect

	mu          sync.Mutex
	receivedLSN uint64
	flushedLSN  uint64 // highest LSN durably persisted (fd.Sync completed); updated only in handleBarrier
	cond        *sync.Cond

	connMu      sync.Mutex // protects activeConns
	activeConns map[net.Conn]struct{}

	dataListener net.Listener
	ctrlListener net.Listener
	stopCh       chan struct{}
	stopped      bool
	wg           sync.WaitGroup
}

const defaultBarrierTimeout = 5 * time.Second

// NewReplicaReceiver creates and starts listening on the data and control ports.
// advertisedHost is the canonical IP for this server (from VS listen addr or
// heartbeat identity). If empty, DataAddr()/CtrlAddr() will fall back to
// outbound-IP detection. On multi-NIC hosts, always provide advertisedHost.
func NewReplicaReceiver(vol *BlockVol, dataAddr, ctrlAddr string, advertisedHost ...string) (*ReplicaReceiver, error) {
	dataLn, err := net.Listen("tcp", dataAddr)
	if err != nil {
		return nil, fmt.Errorf("replica: listen data %s: %w", dataAddr, err)
	}
	ctrlLn, err := net.Listen("tcp", ctrlAddr)
	if err != nil {
		dataLn.Close()
		return nil, fmt.Errorf("replica: listen ctrl %s: %w", ctrlAddr, err)
	}

	var advHost string
	if len(advertisedHost) > 0 {
		advHost = advertisedHost[0]
	}
	// Initialize from the volume's persisted state on receiver recreation.
	// receivedLSN: highest LSN in the WAL (may include unflushed entries).
	// flushedLSN: only checkpoint LSN is guaranteed durable (fd.Sync completed
	// during flusher cycle). Using nextLSN would overstate durability because
	// applyEntry advances nextLSN before barrier fd.Sync.
	initReceived := uint64(0)
	if vol.nextLSN.Load() > 1 {
		initReceived = vol.nextLSN.Load() - 1
	}
	if cfg, progress, ok := vol.ActiveRebuildSession(); ok &&
		(progress.Phase == RebuildPhaseRunning || progress.Phase == RebuildPhaseBaseComplete) &&
		cfg.BaseLSN > initReceived {
		initReceived = cfg.BaseLSN
	}
	initFlushed := uint64(0)
	if vol.flusher != nil {
		initFlushed = vol.flusher.CheckpointLSN()
	}
	r := &ReplicaReceiver{
		vol:            vol,
		receivedLSN:    initReceived,
		flushedLSN:     initFlushed,
		barrierTimeout: defaultBarrierTimeout,
		advertisedHost: advHost,
		dataListener:   dataLn,
		ctrlListener:   ctrlLn,
		stopCh:         make(chan struct{}),
		activeConns:    make(map[net.Conn]struct{}),
	}
	r.cond = sync.NewCond(&r.mu)
	return r, nil
}

// Serve starts accept loops for both listeners. Call Stop() to shut down.
func (r *ReplicaReceiver) Serve() {
	r.wg.Add(2)
	go r.acceptDataLoop()
	go r.acceptCtrlLoop()
}

// Stop shuts down both listeners, closes active connections, and waits for goroutines.
func (r *ReplicaReceiver) Stop() {
	r.mu.Lock()
	if r.stopped {
		r.mu.Unlock()
		return
	}
	r.stopped = true
	r.mu.Unlock()

	close(r.stopCh)
	r.dataListener.Close()
	r.ctrlListener.Close()

	// Close all active connections to unblock ReadFrame calls.
	r.connMu.Lock()
	for conn := range r.activeConns {
		conn.Close()
	}
	r.connMu.Unlock()

	// Wake any barrier waiters so they can exit (must hold mu for cond).
	r.mu.Lock()
	r.cond.Broadcast()
	r.mu.Unlock()
	r.wg.Wait()
}

func (r *ReplicaReceiver) trackConn(conn net.Conn) {
	r.connMu.Lock()
	r.activeConns[conn] = struct{}{}
	r.connMu.Unlock()
}

func (r *ReplicaReceiver) untrackConn(conn net.Conn) {
	r.connMu.Lock()
	delete(r.activeConns, conn)
	r.connMu.Unlock()
}

func (r *ReplicaReceiver) acceptDataLoop() {
	defer r.wg.Done()
	for {
		conn, err := r.dataListener.Accept()
		if err != nil {
			select {
			case <-r.stopCh:
				return
			default:
				log.Printf("replica: data accept error: %v", err)
				return
			}
		}
		r.trackConn(conn)
		r.wg.Add(1)
		go func() {
			defer r.wg.Done()
			defer r.untrackConn(conn)
			r.handleDataConn(conn)
		}()
	}
}

func (r *ReplicaReceiver) acceptCtrlLoop() {
	defer r.wg.Done()
	for {
		conn, err := r.ctrlListener.Accept()
		if err != nil {
			select {
			case <-r.stopCh:
				return
			default:
				log.Printf("replica: ctrl accept error: %v", err)
				return
			}
		}
		r.trackConn(conn)
		r.wg.Add(1)
		go func() {
			defer r.wg.Done()
			defer r.untrackConn(conn)
			r.handleControlConn(conn)
		}()
	}
}

// handleDataConn reads WAL entry frames and applies them to the local volume.
func (r *ReplicaReceiver) handleDataConn(conn net.Conn) {
	defer conn.Close()
	for {
		select {
		case <-r.stopCh:
			return
		default:
		}

		msgType, payload, err := ReadFrame(conn)
		if err != nil {
			select {
			case <-r.stopCh:
			default:
				log.Printf("replica: data read error: %v", err)
			}
			return
		}

		switch msgType {
		case MsgWALEntry:
			if err := r.applyEntry(payload); err != nil {
				log.Printf("replica: apply entry error: %v", err)
			}
		case MsgResumeShipReq:
			r.handleResumeShipReq(conn, payload)
		case MsgCatchupDone:
			lsn, err := DecodeCatchupDone(payload)
			if err != nil {
				log.Printf("replica: decode catchup done: %v", err)
			} else {
				log.Printf("replica: catch-up done, snapshot LSN=%d", lsn)
			}
		default:
			log.Printf("replica: unexpected data message type 0x%02x", msgType)
		}
	}
}

// handleResumeShipReq processes the reconnect handshake from the primary.
func (r *ReplicaReceiver) handleResumeShipReq(conn net.Conn, payload []byte) {
	req, err := DecodeResumeShipReq(payload)
	if err != nil {
		log.Printf("replica: decode resume ship req: %v", err)
		resp := EncodeResumeShipResp(ResumeShipResp{Status: ResumeNeedsRebuild})
		WriteFrame(conn, MsgResumeShipResp, resp)
		return
	}

	// Validate epoch.
	localEpoch := r.vol.epoch.Load()
	if req.Epoch != localEpoch {
		log.Printf("replica: resume ship epoch mismatch: req=%d local=%d", req.Epoch, localEpoch)
		resp := EncodeResumeShipResp(ResumeShipResp{
			Status:            ResumeEpochMismatch,
			ReplicaFlushedLSN: r.resumeRecoverableLSN(req.Epoch),
		})
		WriteFrame(conn, MsgResumeShipResp, resp)
		return
	}

	resp := EncodeResumeShipResp(ResumeShipResp{
		Status:            ResumeOK,
		ReplicaFlushedLSN: r.resumeRecoverableLSN(req.Epoch),
	})
	WriteFrame(conn, MsgResumeShipResp, resp)
}

// resumeRecoverableLSN returns the replica boundary the primary may safely use
// as the reconnect/catch-up starting point.
//
// Normally this is the replica's durable flushed boundary. During an active
// rebuild session, the trusted base already covers BaseLSN and the live WAL
// lane may continue from max(BaseLSN, WALAppliedLSN).
func (r *ReplicaReceiver) resumeRecoverableLSN(epoch uint64) uint64 {
	resumeLSN := r.FlushedLSN()
	if cfg, progress, ok := r.vol.ActiveRebuildSession(); ok &&
		cfg.Epoch == epoch &&
		(progress.Phase == RebuildPhaseRunning || progress.Phase == RebuildPhaseBaseComplete) {
		if progress.WALAppliedLSN > resumeLSN {
			resumeLSN = progress.WALAppliedLSN
		}
		if cfg.BaseLSN > resumeLSN {
			resumeLSN = cfg.BaseLSN
		}
	}
	return resumeLSN
}

// applyEntry decodes and applies a single WAL entry to the local volume.
// The entire apply (LSN check -> WAL append -> dirty map -> receivedLSN update)
// is serialized under mu to prevent TOCTOU races between concurrent entries.
func (r *ReplicaReceiver) applyEntry(payload []byte) error {
	entry, err := DecodeWALEntry(payload)
	if err != nil {
		return fmt.Errorf("decode WAL entry: %w", err)
	}

	// ioMu.RLock: protect WAL/dirtyMap mutation against exclusive restore/import.
	r.vol.ioMu.RLock()
	defer r.vol.ioMu.RUnlock()

	// Validate epoch: replicas must NOT accept epoch bumps from WAL stream.
	// Only the master can change epochs (via SetEpoch in CP3).
	localEpoch := r.vol.epoch.Load()
	if entry.Epoch != localEpoch {
		return fmt.Errorf("%w: entry epoch %d != local %d", ErrStaleEpoch, entry.Epoch, localEpoch)
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	// Enforce contiguous LSN: only accept the next expected entry.
	// This prevents gaps that would let a barrier pass incorrectly.
	if entry.LSN <= r.receivedLSN {
		log.Printf("replica: skipping duplicate/old LSN %d (received %d)", entry.LSN, r.receivedLSN)
		return nil
	}
	if entry.LSN != r.receivedLSN+1 {
		return fmt.Errorf("%w: expected LSN %d, got %d (gap)", ErrDuplicateLSN, r.receivedLSN+1, entry.LSN)
	}

	routedToRebuild := false
	if cfg, progress, ok := r.vol.ActiveRebuildSession(); ok &&
		cfg.Epoch == entry.Epoch &&
		(progress.Phase == RebuildPhaseRunning || progress.Phase == RebuildPhaseBaseComplete) {
		if err := r.vol.ApplyRebuildSessionWALEntry(cfg.SessionID, &entry); err != nil {
			return fmt.Errorf("rebuild session WAL apply: %w", err)
		}
		routedToRebuild = true
	}

	if !routedToRebuild {
		// Append to local WAL (with retry on WAL full).
		walOff, err := r.replicaAppendWithRetry(&entry)
		if err != nil {
			return fmt.Errorf("WAL append: %w", err)
		}

		// Update dirty map.
		switch entry.Type {
		case EntryTypeWrite, EntryTypeTrim:
			blocks := entry.Length / r.vol.super.BlockSize
			for i := uint32(0); i < blocks; i++ {
				r.vol.dirtyMap.Put(entry.LBA+uint64(i), walOff, entry.LSN, r.vol.super.BlockSize)
			}
		}
	}

	// Update receivedLSN and signal barrier waiters.
	r.receivedLSN = entry.LSN
	r.cond.Broadcast()

	// Update vol.nextLSN so Status().WALHeadLSN reflects replicated state.
	// CAS loop: only advance, never regress.
	for {
		cur := r.vol.nextLSN.Load()
		next := entry.LSN + 1
		if next <= cur {
			break
		}
		if r.vol.nextLSN.CompareAndSwap(cur, next) {
			break
		}
	}

	return nil
}

// replicaAppendWithRetry appends a WAL entry, retrying on WAL-full by
// triggering the flusher. Caller must hold r.mu.
func (r *ReplicaReceiver) replicaAppendWithRetry(entry *WALEntry) (uint64, error) {
	walOff, err := r.vol.wal.Append(entry)
	if !errors.Is(err, ErrWALFull) {
		return walOff, err
	}

	deadline := time.After(r.vol.config.WALFullTimeout)
	for errors.Is(err, ErrWALFull) {
		select {
		case <-r.stopCh:
			return 0, fmt.Errorf("replica: stopped during WAL retry")
		default:
		}
		if r.vol.flusher != nil {
			r.vol.flusher.NotifyUrgent()
		}
		// Release mu briefly so barrier waiters can proceed and
		// the flusher can make progress (it may need dirty map lock).
		r.mu.Unlock()
		select {
		case <-deadline:
			r.mu.Lock()
			return 0, fmt.Errorf("replica: WAL full timeout: %w", ErrWALFull)
		case <-time.After(1 * time.Millisecond):
		}
		r.mu.Lock()
		walOff, err = r.vol.wal.Append(entry)
	}
	return walOff, err
}

// ApplyEntryForTest encodes and applies a WAL entry directly. Test-only.
func (r *ReplicaReceiver) ApplyEntryForTest(entry *WALEntry) error {
	encoded, err := entry.Encode()
	if err != nil {
		return err
	}
	return r.applyEntry(encoded)
}

// ReceivedLSN returns the highest LSN received and written to the local WAL.
func (r *ReplicaReceiver) ReceivedLSN() uint64 {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.receivedLSN
}

// FlushedLSN returns the highest LSN durably persisted on this replica
// (after successful WAL fd.Sync). Updated only by handleBarrier.
func (r *ReplicaReceiver) FlushedLSN() uint64 {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.flushedLSN
}

// DataAddr returns the data listener's canonical address (ip:port).
// Wildcard listener addresses are resolved using the advertised host
// or outbound-IP fallback.
func (r *ReplicaReceiver) DataAddr() string {
	return canonicalizeListenerAddr(r.dataListener.Addr(), r.advertisedHost)
}

// CtrlAddr returns the control listener's canonical address (ip:port).
func (r *ReplicaReceiver) CtrlAddr() string {
	return canonicalizeListenerAddr(r.ctrlListener.Addr(), r.advertisedHost)
}