seaweedfs/weed/mount/peer_fetcher.go

package mount

import (
	"bytes"
	"context"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"sort"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// errPeerReadSkipped signals that the read didn't go through the peer
// path for a benign reason (local cache hit, no peer owner, etc.) rather
// than a genuine failure. Callers fall through to the volume path just
// as they would for a real error, but suppress the "peer read failed"
// log that would otherwise mislead operators into thinking peer sharing
// is broken.
var errPeerReadSkipped = fmt.Errorf("peer read skipped (not an error)")

// peerLookupTimeout bounds the ChunkLookup RPC. Short because the result
// is consumed on the read critical path.
const peerLookupTimeout = 500 * time.Millisecond

// peerFetchTimeout bounds a single FetchChunk stream; on expiry we fall
// through to the next holder or the volume server.
const peerFetchTimeout = 5 * time.Second

// maxPeerFetchChunkBytes caps how much we will accept from a single
// FetchChunk stream. gRPC per-message size is already capped by the
// server option; this is belt-and-suspenders against a runaway peer.
const maxPeerFetchChunkBytes = 64 * 1024 * 1024

// tryPeerRead attempts to satisfy a read from a peer mount's chunk cache.
// Returns (bytesRead, modifiedTsNs, nil) on success. On any failure it
// returns (0, 0, err) so the caller falls through to
// entryChunkGroup.ReadDataAt (the volume-server path).
//
// Flow:
//  1. Resolve the offset's leaf chunk (flattening manifests).
//  2. Ask the HRW owner mount for current holders via ChunkLookup.
//  3. For each holder (LRU order from PR #5), open a FetchChunk stream,
//     assemble frames into a size-bounded buffer, and verify MD5
//     end-to-end against FileChunk.ETag.
//  4. On success, populate chunk_cache and enqueue an announce so
//     other mounts can discover us as a new holder.
//
// chunks is a snapshot captured under the LockedEntry lock by the caller.
func (fh *FileHandle) tryPeerRead(ctx context.Context, fileSize int64, buff []byte, offset int64, chunks []*filer_pb.FileChunk) (int64, int64, error) {
	if fh.wfs.peerRegistrar == nil || fh.wfs.peerConnPool == nil {
		return 0, 0, fmt.Errorf("peer sharing not configured")
	}

	// Resolve offset → leaf chunk, flattening any manifest indirection.
	readStop := offset + int64(len(buff))
	if readStop > fileSize {
		readStop = fileSize
	}
	dataChunks, _, err := filer.ResolveChunkManifest(ctx, fh.wfs.LookupFn(), chunks, offset, readStop, fh.wfs.CacheInvalidator())
	if err != nil {
		return 0, 0, fmt.Errorf("resolve manifest: %w", err)
	}
	targetChunk, chunkOffset := findChunkContaining(dataChunks, offset)
	if targetChunk == nil {
		return 0, 0, fmt.Errorf("no leaf chunk for offset %d", offset)
	}
	// Reject reads that cross a chunk boundary. We only fetch one chunk
	// here, and the caller (readFromChunks) maps a short non-error
	// return to "success, zero-fill the rest" — which would silently
	// corrupt reads that should actually span two chunks. Bail so the
	// fallback ReadDataAt path can handle the multi-chunk case
	// correctly. See weedfs_file_read.go short-read semantics.
	chunkEnd := targetChunk.Offset + int64(targetChunk.Size)
	if readStop > chunkEnd {
		return 0, 0, errPeerReadSkipped
	}

	// Fail fast when the chunk is already cached locally: the fallback
	// ReadDataAt path will satisfy the read from chunkCache with no RPCs,
	// so dialing a peer (ChunkLookup + FetchChunk) would be pure overhead.
	if fh.wfs.chunkCache != nil && fh.wfs.chunkCache.IsInCache(targetChunk.FileId, true) {
		return 0, 0, errPeerReadSkipped
	}

	selfAddr := ""
	if fh.wfs.peerGrpcServer != nil {
		selfAddr = fh.wfs.peerGrpcServer.SelfAddr()
	}

	owner := fh.wfs.peerRegistrar.OwnerFor(targetChunk.FileId)
	if owner == "" || owner == selfAddr {
		return 0, 0, fmt.Errorf("no peer owner for fid %s", targetChunk.FileId)
	}

	holders, err := peerLookupHolders(ctx, fh.wfs.peerConnPool.Dialer(), owner, targetChunk.FileId)
	if err != nil {
		return 0, 0, fmt.Errorf("peer lookup: %w", err)
	}
	if len(holders) == 0 {
		return 0, 0, fmt.Errorf("no peer holder for fid %s", targetChunk.FileId)
	}

	// Re-rank holders by locality (same rack > same DC > elsewhere), keeping
	// the server's LRU order stable within each bucket. The server caps the
	// list at maxLookupHolders, so this is always a small N.
	sortHoldersByLocality(holders, fh.wfs.option.PeerDataCenter, fh.wfs.option.PeerRack)

	dial := fh.wfs.peerConnPool.Dialer()
	for _, h := range holders {
		if h.addr == selfAddr {
			continue
		}
		data, ferr := fetchChunkFromPeer(ctx, dial, h.addr, targetChunk.FileId, targetChunk.Size, targetChunk.ETag)
		if ferr != nil {
			glog.V(2).Infof("peer-fetch %s from %s: %v", targetChunk.FileId, h.addr, ferr)
			continue
		}
		if fh.wfs.chunkCache != nil {
			fh.wfs.chunkCache.SetChunk(targetChunk.FileId, data)
		}
		if fh.wfs.peerAnnouncer != nil {
			fh.wfs.peerAnnouncer.EnqueueAnnounce(targetChunk.FileId)
		}
		if chunkOffset >= int64(len(data)) {
			return 0, 0, fmt.Errorf("peer returned short chunk")
		}
		// Cap the copy to whichever is smaller: the caller's buffer, or
		// the remaining bytes before logical EOF (readStop already
		// clamped to fileSize). FileChunk.Size can legitimately exceed
		// the logical file length when the last chunk is partially
		// written and the filer stored the full padded buffer, so a
		// naïve copy(buff, data[chunkOffset:]) would return bytes past
		// EOF. The caller treats a short non-error return as success
		// and zero-fills the tail, so returning too many bytes here is
		// a correctness bug, not just wasted I/O.
		remaining := readStop - offset
		available := int64(len(data)) - chunkOffset
		maxCopy := int64(len(buff))
		if remaining < maxCopy {
			maxCopy = remaining
		}
		if available < maxCopy {
			maxCopy = available
		}
		copied := copy(buff[:maxCopy], data[chunkOffset:chunkOffset+maxCopy])
		return int64(copied), targetChunk.ModifiedTsNs, nil
	}
	return 0, 0, fmt.Errorf("no peer served fid %s", targetChunk.FileId)
}

func findChunkContaining(chunks []*filer_pb.FileChunk, offset int64) (*filer_pb.FileChunk, int64) {
	for _, c := range chunks {
		start := c.Offset
		stop := c.Offset + int64(c.Size)
		if offset >= start && offset < stop {
			return c, offset - start
		}
	}
	return nil, 0
}

// peerHolder is a holder entry carried through the fetcher: addr for
// dialing plus the DC/Rack labels the owner recorded at announce time, so
// the fetcher can re-rank by its own locality before dialing.
type peerHolder struct {
	addr string
	dc   string
	rack string
}

// peerLookupHolders calls ChunkLookup on the given HRW owner and returns
// the holders in the server-reported (LRU) order along with their locality
// labels. The pooled dialer is passed in from the caller.
func peerLookupHolders(ctx context.Context, dial MountPeerDialer, ownerAddr, fid string) ([]peerHolder, error) {
	client, closeFn, err := dial(ctx, ownerAddr)
	if err != nil {
		return nil, err
	}
	defer closeFn()

	callCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
	defer cancel()
	resp, err := client.ChunkLookup(callCtx, &mount_peer_pb.ChunkLookupRequest{FileIds: []string{fid}})
	if err != nil {
		return nil, err
	}
	set, ok := resp.PeersByFid[fid]
	if !ok || set == nil {
		return nil, nil
	}
	out := make([]peerHolder, 0, len(set.Peers))
	for _, p := range set.Peers {
		out = append(out, peerHolder{addr: p.PeerAddr, dc: p.DataCenter, rack: p.Rack})
	}
	return out, nil
}

// localityBucket scores how "close" a peer is to self. Lower is better:
//
//	0 = same rack in same DC      (shortest hop)
//	1 = same DC, different rack   (cross-rack but still intra-DC)
//	2 = different DC, or unknown  (anything else)
//
// Missing labels on either side fall to bucket 2 — we don't claim locality
// we can't prove.
func localityBucket(selfDC, selfRack, peerDC, peerRack string) int {
	if selfDC == "" || peerDC == "" || selfDC != peerDC {
		return 2
	}
	if selfRack != "" && peerRack != "" && selfRack == peerRack {
		return 0
	}
	return 1
}

// sortHoldersByLocality stable-sorts holders so the most local peers are
// tried first. The server-returned order is LRU (freshest holder first),
// and sort.SliceStable preserves that ordering within each locality
// bucket, so among equally-local peers we still prefer the freshest.
func sortHoldersByLocality(holders []peerHolder, selfDC, selfRack string) {
	if len(holders) < 2 {
		return
	}
	sort.SliceStable(holders, func(i, j int) bool {
		return localityBucket(selfDC, selfRack, holders[i].dc, holders[i].rack) <
			localityBucket(selfDC, selfRack, holders[j].dc, holders[j].rack)
	})
}

// fetchChunkFromPeer server-streams a chunk from the given peer and
// verifies the MD5 of the assembled bytes against expectedETag.
//
// expectedSize (when > 0) is the authoritative chunk length from the
// filer entry; we pre-allocate exactly that to avoid slice growth and
// reject streams that overshoot it. A zero expectedSize falls back to
// maxPeerFetchChunkBytes as a safety ceiling.
func fetchChunkFromPeer(ctx context.Context, dial MountPeerDialer, peerAddr, fid string, expectedSize uint64, expectedETag string) ([]byte, error) {
	client, closeFn, err := dial(ctx, peerAddr)
	if err != nil {
		return nil, err
	}
	defer closeFn()

	callCtx, cancel := context.WithTimeout(ctx, peerFetchTimeout)
	defer cancel()

	stream, err := client.FetchChunk(callCtx, &mount_peer_pb.FetchChunkRequest{
		FileId:       fid,
		ExpectedEtag: expectedETag,
		ExpectedSize: expectedSize,
	})
	if err != nil {
		return nil, err
	}

	// capHint pre-sizes the assembly buffer from the filer-reported chunk
	// size. When the caller didn't know the size (expectedSize == 0), we
	// let append grow the buffer rather than reserve the 64 MiB ceiling —
	// typical chunks are a few MiB, and the maxPeerFetchChunkBytes check
	// during Recv is the real safety ceiling.
	capHint := expectedSize
	if capHint > maxPeerFetchChunkBytes {
		capHint = maxPeerFetchChunkBytes
	}
	buf := make([]byte, 0, capHint)

	for {
		resp, rerr := stream.Recv()
		if rerr == io.EOF {
			break
		}
		if rerr != nil {
			if status.Code(rerr) == codes.NotFound {
				return nil, fmt.Errorf("peer not cached")
			}
			return nil, rerr
		}
		if len(buf)+len(resp.Data) > int(maxPeerFetchChunkBytes) {
			return nil, fmt.Errorf("peer response exceeds max chunk size %d", maxPeerFetchChunkBytes)
		}
		buf = append(buf, resp.Data...)
	}

	if expectedSize > 0 && uint64(len(buf)) != expectedSize {
		return nil, fmt.Errorf("peer returned %d bytes, expected %d", len(buf), expectedSize)
	}

	// Per-chunk integrity check — the peer is treated as untrusted.
	// FileChunk.ETag is UploadResult.ContentMd5, which is base64 of the
	// raw 16-byte MD5 (see weed/operation/upload_content.go:64 and
	// filer's ETagChunks which decodes via util.Base64Md5ToBytes).
	// Compare raw digests so we don't reject every valid peer response
	// because of an encoding mismatch.
	if expectedETag != "" {
		got := md5.Sum(buf)
		if !etagMatchesMD5(expectedETag, got[:]) {
			return nil, fmt.Errorf("etag mismatch: peer=%s got=%x want=%s", peerAddr, got, expectedETag)
		}
	}
	return buf, nil
}

// etagMatchesMD5 compares a stored FileChunk.ETag string to a raw 16-byte
// MD5 digest. FileChunk.ETag is produced as base64(md5) on upload, but
// some older chunks (and tests) use hex(md5). Accept either.
func etagMatchesMD5(etag string, rawMD5 []byte) bool {
	if len(rawMD5) != md5.Size {
		return false
	}
	if dec := util.Base64Md5ToBytes(etag); len(dec) == md5.Size {
		return bytes.Equal(dec, rawMD5)
	}
	if dec, err := hex.DecodeString(etag); err == nil && len(dec) == md5.Size {
		return bytes.Equal(dec, rawMD5)
	}
	return false
}