seaweedfs/weed/server/volume_grpc_remote.go

package weed_server

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
	azureremote "github.com/seaweedfs/seaweedfs/weed/remote_storage/azure"
	gcsremote "github.com/seaweedfs/seaweedfs/weed/remote_storage/gcs"
	s3remote "github.com/seaweedfs/seaweedfs/weed/remote_storage/s3"
	"github.com/seaweedfs/seaweedfs/weed/security"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/util"
	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)

// lookupIPAddrFunc resolves a host to one or more IP addresses. It is a
// package-level variable so tests can substitute a deterministic resolver.
var lookupIPAddrFunc = net.DefaultResolver.LookupIPAddr

// blockedIMDSHosts lists hostnames that target cloud instance metadata
// services (IMDS). These are blocked regardless of how they happen to
// resolve, because some environments alias the IMDS address under a name.
var blockedIMDSHosts = map[string]struct{}{
	"metadata.google.internal": {},
	"metadata":                 {},
}

// validateRemoteEndpoint returns an error if the supplied S3 endpoint is not
// safe to dial from a server that has network access to cluster-internal
// hosts. It rejects empty/non-http(s) schemes, loopback/link-local/
// unspecified addresses, RFC 1918 + CGNAT ranges, and well-known IMDS
// hostnames. Operators that legitimately fetch from private hosts can opt
// out with -volume.allowUntrustedRemoteEndpoints.
func validateRemoteEndpoint(ctx context.Context, endpoint string) error {
	if strings.TrimSpace(endpoint) == "" {
		return fmt.Errorf("remote endpoint is empty")
	}
	u, parseErr := url.Parse(endpoint)
	if parseErr != nil {
		return fmt.Errorf("parse remote endpoint %q: %w", endpoint, parseErr)
	}
	scheme := strings.ToLower(u.Scheme)
	if scheme != "http" && scheme != "https" {
		return fmt.Errorf("remote endpoint %q must use http or https, got %q", endpoint, u.Scheme)
	}
	host := u.Hostname()
	if host == "" {
		return fmt.Errorf("remote endpoint %q has no host", endpoint)
	}
	lowerHost := strings.ToLower(host)
	if _, ok := blockedIMDSHosts[lowerHost]; ok {
		return fmt.Errorf("remote endpoint %q targets instance metadata service", endpoint)
	}
	if ip := net.ParseIP(host); ip != nil {
		if err := checkBlockedIP(endpoint, ip); err != nil {
			return err
		}
		return nil
	}
	resolveCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()
	addrs, lookupErr := lookupIPAddrFunc(resolveCtx, host)
	if lookupErr != nil {
		return fmt.Errorf("resolve remote endpoint host %q: %w", host, lookupErr)
	}
	for _, addr := range addrs {
		if err := checkBlockedIP(endpoint, addr.IP); err != nil {
			return err
		}
	}
	return nil
}

// imdsIPv4 is the AWS/Azure/GCP IPv4 IMDS address. It is link-local and is
// already covered by IsLinkLocalUnicast, but is named explicitly so the
// error message is unambiguous in logs.
var imdsIPv4 = net.ParseIP("169.254.169.254")

// cgnatNet is the RFC 6598 carrier-grade NAT range (100.64.0.0/10). The
// stdlib's IsPrivate covers RFC 1918 but not CGNAT, so check it explicitly.
var cgnatNet = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}

func checkBlockedIP(endpoint string, ip net.IP) error {
	return checkBlockedIPPolicy(endpoint, ip, false)
}

// checkBlockedIPPolicy rejects addresses that must never be dialed from a
// server with cluster-internal reach. allowPrivate keeps RFC 1918 / CGNAT
// reachable for callers whose target legitimately sits on an internal network
// (peer volume servers), while still blocking loopback, link-local (IMDS) and
// unspecified.
func checkBlockedIPPolicy(endpoint string, ip net.IP, allowPrivate bool) error {
	if ip == nil {
		return nil
	}
	if ip.Equal(imdsIPv4) {
		return fmt.Errorf("remote endpoint %q targets instance metadata service %s", endpoint, ip)
	}
	switch {
	case ip.IsLoopback():
		return fmt.Errorf("remote endpoint %q resolves to loopback address %s", endpoint, ip)
	case ip.IsUnspecified():
		return fmt.Errorf("remote endpoint %q resolves to unspecified address %s", endpoint, ip)
	case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
		return fmt.Errorf("remote endpoint %q resolves to link-local address %s", endpoint, ip)
	case ip.IsInterfaceLocalMulticast():
		return fmt.Errorf("remote endpoint %q resolves to interface-local address %s", endpoint, ip)
	}
	if !allowPrivate {
		switch {
		case ip.IsPrivate():
			return fmt.Errorf("remote endpoint %q resolves to private address %s", endpoint, ip)
		case cgnatNet.Contains(ip):
			return fmt.Errorf("remote endpoint %q resolves to CGNAT address %s", endpoint, ip)
		}
	}
	// IPv6 transition addresses embed an IPv4 destination that routes to the
	// same host wherever the matching relay exists (common in IPv6-only cloud).
	// net.IP only normalizes ::ffff: mapped addresses, so pull the embedded
	// IPv4 out of the other forms and re-check it against the deny list.
	if embedded := embeddedTransitionIPv4(ip); embedded != nil {
		return checkBlockedIPPolicy(endpoint, embedded, allowPrivate)
	}
	return nil
}

// validateReplicaTarget rejects a peer volume server address that could
// redirect a dial away from the cluster: replica upload targets and the
// copy/tail source addresses are all caller-supplied. The target must be a
// bare host:port -- a scheme, userinfo, path, query or fragment can smuggle a
// different destination through fmt.Sprintf -- whose host is not loopback,
// link-local (IMDS) or unspecified. Cluster peers legitimately sit on private
// networks, so RFC 1918 / CGNAT are allowed.
func validateReplicaTarget(ctx context.Context, target string) error {
	if strings.TrimSpace(target) == "" {
		return fmt.Errorf("replica target is empty")
	}
	if strings.Contains(target, "://") || strings.ContainsAny(target, "/?#@\\") {
		return fmt.Errorf("replica target %q must be a bare host:port", target)
	}
	host, _, splitErr := net.SplitHostPort(target)
	if splitErr != nil {
		return fmt.Errorf("replica target %q must be a bare host:port: %w", target, splitErr)
	}
	if host == "" {
		return fmt.Errorf("replica target %q has no host", target)
	}
	if _, ok := blockedIMDSHosts[strings.ToLower(host)]; ok {
		return fmt.Errorf("replica target %q targets instance metadata service", target)
	}
	if ip := net.ParseIP(host); ip != nil {
		return checkBlockedIPPolicy(target, ip, true)
	}
	resolveCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()
	addrs, lookupErr := lookupIPAddrFunc(resolveCtx, host)
	if lookupErr != nil {
		return fmt.Errorf("resolve replica target host %q: %w", host, lookupErr)
	}
	for _, addr := range addrs {
		if err := checkBlockedIPPolicy(target, addr.IP, true); err != nil {
			return err
		}
	}
	return nil
}

// embeddedTransitionIPv4 returns the IPv4 address carried by an IPv6 transition
// address -- NAT64 64:ff9b::/96 (RFC 6052), 6to4 2002::/16 (RFC 3056), Teredo
// 2001:0000::/32 (RFC 4380), and the deprecated IPv4-compatible ::/96 (RFC
// 4291) -- or nil when ip is not one of those. IPv4-mapped ::ffff:0:0/96 is
// excluded because net.IP already normalizes it via To4.
func embeddedTransitionIPv4(ip net.IP) net.IP {
	v6 := ip.To16()
	if v6 == nil || ip.To4() != nil {
		return nil
	}
	switch {
	case v6[0] == 0x00 && v6[1] == 0x64 && v6[2] == 0xff && v6[3] == 0x9b && allZero(v6[4:12]):
		return net.IPv4(v6[12], v6[13], v6[14], v6[15])
	case v6[0] == 0x20 && v6[1] == 0x02:
		return net.IPv4(v6[2], v6[3], v6[4], v6[5])
	case v6[0] == 0x20 && v6[1] == 0x01 && v6[2] == 0x00 && v6[3] == 0x00:
		// Teredo obfuscates the client IPv4 as its ones' complement.
		return net.IPv4(v6[12]^0xff, v6[13]^0xff, v6[14]^0xff, v6[15]^0xff)
	case allZero(v6[:12]):
		// IPv4-compatible ::a.b.c.d; :: and ::1 are already handled above.
		return net.IPv4(v6[12], v6[13], v6[14], v6[15])
	}
	return nil
}

func allZero(b []byte) bool {
	for _, c := range b {
		if c != 0 {
			return false
		}
	}
	return true
}

// guardedDialer returns a DialContext that resolves the host itself and
// re-applies checkBlockedIP to every resolved address immediately before
// dialing. This closes the DNS-rebinding window between
// validateRemoteEndpoint and the actual TCP connect performed by the AWS S3
// client: even if the attacker's DNS flips to 127.0.0.1 (or any other
// blocked range) after the up-front check, the dial is refused.
func guardedDialer(endpoint string) func(ctx context.Context, network, addr string) (net.Conn, error) {
	return guardedDialerPolicy(endpoint, false)
}

// guardedDialerPolicy is guardedDialer with the same allowPrivate knob as
// checkBlockedIPPolicy, so the replica upload path can keep dialing private
// peers while still refusing loopback / link-local / unspecified at connect
// time (closing the rebinding window for replica hostnames too).
func guardedDialerPolicy(endpoint string, allowPrivate bool) func(ctx context.Context, network, addr string) (net.Conn, error) {
	return func(ctx context.Context, network, addr string) (net.Conn, error) {
		host, port, splitErr := net.SplitHostPort(addr)
		if splitErr != nil {
			return nil, splitErr
		}
		// If the host is already a literal IP just validate and dial it.
		if ip := net.ParseIP(host); ip != nil {
			if err := checkBlockedIPPolicy(endpoint, ip, allowPrivate); err != nil {
				return nil, err
			}
			return util.OutboundDialContext(ctx, network, addr)
		}
		// Otherwise resolve, validate every answer, and dial the first IP
		// that passes the deny list. Using a literal-IP target prevents the
		// kernel resolver in net.Dialer from looking the name up a second
		// time inside Dial and getting a different answer.
		addrs, lookupErr := lookupIPAddrFunc(ctx, host)
		if lookupErr != nil {
			return nil, fmt.Errorf("resolve remote endpoint host %q: %w", host, lookupErr)
		}
		var firstBlockErr error
		for _, a := range addrs {
			if err := checkBlockedIPPolicy(endpoint, a.IP, allowPrivate); err != nil {
				if firstBlockErr == nil {
					firstBlockErr = err
				}
				continue
			}
			return util.OutboundDialContext(ctx, network, net.JoinHostPort(a.IP.String(), port))
		}
		if firstBlockErr != nil {
			return nil, firstBlockErr
		}
		return nil, fmt.Errorf("resolve remote endpoint host %q: no addresses", host)
	}
}

// guardedGrpcDialOption returns a grpc.DialOption that re-applies the replica
// deny list to every resolved address at connect time, pinning a validated
// copy/tail source against DNS rebinding. It is nil when the operator opted
// out with AllowUntrustedRemoteEndpoints; pb skips nil dial options.
func (vs *VolumeServer) guardedGrpcDialOption(endpoint string) grpc.DialOption {
	if vs.AllowUntrustedRemoteEndpoints {
		return nil
	}
	dial := guardedDialerPolicy(endpoint, true)
	return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
		return dial(ctx, "tcp", addr)
	})
}

// newGuardedHTTPClient returns an *http.Client whose transport refuses to
// dial addresses that fail checkBlockedIP at connect time. One client serves
// one remote configuration (BuildGuardedRemoteStorageClient caches it per
// configuration); never share it across remote configs.
func newGuardedHTTPClient(endpoint string) *http.Client {
	client := newGuardedHTTPClientPolicy(endpoint, false)
	if t, ok := client.Transport.(*http.Transport); ok {
		tuneSharedTransport(t)
	}
	return client
}

// The shared transport's timeouts: with one client per remote and a cap per
// host, a request that never finishes would hold a slot forever, and enough of
// them stall every later lazy stat, cold read and remote delete of that remote.
// A dial, a TLS handshake and a response's headers each have a deadline, and a
// stalled body an idle one.
const (
	guardedDialTimeout           = 10 * time.Second
	guardedTLSHandshakeTimeout   = 10 * time.Second
	guardedResponseHeaderTimeout = 60 * time.Second
)

// guardedIOIdleTimeout bounds the stall of one exchange: a body that stops
// mid-transfer frees its connection slot while a slow but progressing transfer
// is never cut off. A var so tests can shorten it.
var guardedIOIdleTimeout = 2 * time.Minute

// idleTimeoutConn refreshes a deadline on every read and write: a peer that
// stops sending or receiving is cut off after idle has passed without
// progress.
type idleTimeoutConn struct {
	net.Conn
	idle time.Duration
}

func (c *idleTimeoutConn) Read(b []byte) (int, error) {
	_ = c.SetReadDeadline(time.Now().Add(c.idle))
	return c.Conn.Read(b)
}

func (c *idleTimeoutConn) Write(b []byte) (int, error) {
	_ = c.SetWriteDeadline(time.Now().Add(c.idle))
	return c.Conn.Write(b)
}

// tuneSharedTransport sets what a transport shared by every request to one
// remote needs: enough idle connections for a burst to reuse (Go's default is
// 2 per host), a cap on connections per host (a burst of cold reads queues
// instead of opening hundreds, which an endpoint limiting connections per
// source would refuse), and deadlines, so a hung request frees its slot.
func tuneSharedTransport(t *http.Transport) {
	t.MaxIdleConns = 64
	t.MaxIdleConnsPerHost = 32
	t.MaxConnsPerHost = guardedMaxConnsPerHost()
	t.TLSHandshakeTimeout = guardedTLSHandshakeTimeout
	t.ResponseHeaderTimeout = guardedResponseHeaderTimeout
	if dial := t.DialContext; dial != nil {
		t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
			ctx, cancel := context.WithTimeout(ctx, guardedDialTimeout)
			defer cancel() // an established connection outlives the dial's context
			conn, err := dial(ctx, network, addr)
			if err != nil {
				return nil, err
			}
			return &idleTimeoutConn{Conn: conn, idle: guardedIOIdleTimeout}, nil
		}
	}
}

// guardedMaxConnsPerHost is WEED_REMOTE_MAX_CONNS_PER_HOST, default 64.
func guardedMaxConnsPerHost() int {
	if v := os.Getenv("WEED_REMOTE_MAX_CONNS_PER_HOST"); v != "" {
		if n, err := strconv.Atoi(v); err == nil && n > 0 {
			return n
		}
		glog.Warningf("WEED_REMOTE_MAX_CONNS_PER_HOST=%q is not a positive number; using 64", v)
	}
	return 64
}

// guardedCredentialsVersion names the credentials a guarded client reads when
// it is built that its configuration does not hold, so that a change of them
// builds a new client: the gcs credentials sources of CredentialsVersion, or,
// for an s3-compatible remote, the resolved keys (which a type's environment
// fallback may supply), plus — with a key or secret still missing (the SDK's
// default chain) — the shared credentials and config files and the AWS_*
// variables the chain reads. "" for a remote of another kind.
func guardedCredentialsVersion(conf *remote_pb.RemoteConf) string {
	if conf.GetType() == "gcs" {
		return remote_storage.CredentialsVersion(conf)
	}
	accessKey, secretKey, ok := s3remote.ResolvedCredentials(conf)
	if !ok {
		return ""
	}
	h := sha256.New()
	h.Write([]byte(accessKey))
	h.Write([]byte{0})
	h.Write([]byte(secretKey))
	if s3remote.UsesDefaultCredentialChain(conf) {
		shared := awsSharedFile("AWS_SHARED_CREDENTIALS_FILE", "credentials")
		config := awsSharedFile("AWS_CONFIG_FILE", "config")
		h.Write([]byte{0})
		h.Write([]byte(remote_storage.FileVersion(shared)))
		h.Write([]byte{0})
		h.Write([]byte(remote_storage.FileVersion(config)))
		for _, k := range []string{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE"} {
			h.Write([]byte{0})
			h.Write([]byte(os.Getenv(k)))
		}
	}
	return hex.EncodeToString(h.Sum(nil)[:16])
}

// awsSharedFile is the SDK's effective path of a shared file: the environment
// override or the default under ~/.aws.
func awsSharedFile(env, name string) string {
	if path := os.Getenv(env); path != "" {
		return path
	}
	if home, err := os.UserHomeDir(); err == nil {
		return filepath.Join(home, ".aws", name)
	}
	return ""
}

// newGuardedHTTPClientPolicy is newGuardedHTTPClient with the allowPrivate knob
// for the replica upload path, whose targets are cluster peers on private
// networks.
func newGuardedHTTPClientPolicy(endpoint string, allowPrivate bool) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			// No proxy: guardedDialer must see the real target address. Through
			// a proxy it would only validate the proxy's IP while the proxy
			// re-resolves the endpoint host, reopening the rebinding window the
			// dialer exists to close. Operators that need a proxy can opt out
			// with -volume.allowUntrustedRemoteEndpoints.
			Proxy:                 nil,
			DialContext:           guardedDialerPolicy(endpoint, allowPrivate),
			ForceAttemptHTTP2:     true,
			MaxIdleConns:          16,
			IdleConnTimeout:       60 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
}

// newGuardedPeerHTTPClient is newGuardedHTTPClientPolicy for uploads to cluster
// peers: a volume server fronted by TLS expects the https.client configuration
// the shared global client carries, so the guarded transport borrows it along
// with its redirect policy — a redirect must not downgrade an upload carrying
// data and auth headers to plaintext.
func newGuardedPeerHTTPClient(endpoint string) *http.Client {
	globalClient := util_http.GetGlobalHttpClient()
	client := newGuardedHTTPClientPolicy(endpoint, true)
	client.Transport.(*http.Transport).TLSClientConfig =
		globalClient.GetClientTransport().TLSClientConfig
	client.CheckRedirect = globalClient.Client.CheckRedirect
	return client
}

// guardedRemoteClient reports the caller-supplied endpoint a backend dials
// directly and a constructor that routes through the given HTTP client, or
// ok=false when nothing in the conf steers a destination. The S3-SDK family,
// azure (once AzureEndpoint is set) and the gcs token exchange all honor a
// caller-supplied endpoint, so each must pass the SSRF deny-list and the
// rebinding-safe dialer.
func guardedRemoteClient(remoteConf *remote_pb.RemoteConf) (endpoint string, makeClient func(*http.Client) (remote_storage.RemoteStorageClient, error), ok bool) {
	if remoteConf == nil {
		return "", nil, false
	}
	if ep, isS3 := s3remote.S3CompatibleEndpoint(remoteConf); isS3 {
		if ep == "" && remoteConf.Type == "s3" {
			return "", nil, false
		}
		return ep, func(httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
			return s3remote.MakeWithHTTPClient(remoteConf, httpClient)
		}, true
	}
	if remoteConf.Type == "azure" && remoteConf.AzureEndpoint != "" {
		return remoteConf.AzureEndpoint, func(httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
			return azureremote.MakeWithHTTPClient(remoteConf, httpClient)
		}, true
	}
	// gcs reaches a fixed object host, but the token exchange goes wherever the
	// supplied credentials say, so guard that endpoint instead.
	if remoteConf.Type == "gcs" && remoteConf.GcsGoogleApplicationCredentials != "" {
		if data, err := loadGcsCredentialsContent(remoteConf.GcsGoogleApplicationCredentials); err == nil {
			if _, tokenURL, parseErr := gcsremote.ParseInlineCredentials(string(data)); parseErr == nil {
				return tokenURL, func(httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
					return gcsremote.MakeWithHTTPClient(remoteConf, httpClient, gcsremote.StaticKeyCredentialTypes...)
				}, true
			}
		}
	}
	return "", nil, false
}

var errGcsCredentialsUnreadable = errors.New("gcs credentials file is not readable or does not contain valid credentials")

// loadGcsCredentialsContent returns the credential JSON for a gcs credentials
// value, reading from disk when it is a filesystem path (as written by
// remote.configure -gcs.appCredentialsFile). This mirrors what the gcs client
// itself does in MakeWithHTTPClient, so the guard validates the same content
// the client will eventually load.
func loadGcsCredentialsContent(creds string) ([]byte, error) {
	if creds == "" {
		return nil, nil
	}
	if strings.HasPrefix(creds, "{") {
		return []byte(creds), nil
	}
	data, err := os.ReadFile(util.ResolvePath(creds))
	if err != nil {
		return nil, errGcsCredentialsUnreadable
	}
	return data, nil
}

// checkGcsCredentials rejects a caller-supplied gcs credentials value that
// would make the SDK read from somewhere other than the credentials themselves,
// so the request fails before any client is built.
func checkGcsCredentials(creds string) error {
	if creds == "" {
		return nil
	}
	data, err := loadGcsCredentialsContent(creds)
	if err != nil {
		return err
	}
	credType, _, parseErr := gcsremote.ParseInlineCredentials(string(data))
	if parseErr != nil {
		return parseErr
	}
	if !slices.Contains(gcsremote.StaticKeyCredentialTypes, credType) {
		return fmt.Errorf("gcs credential type %q is not accepted here", credType)
	}
	return nil
}

// BuildGuardedRemoteStorageClient builds a remote storage client whose dial
// path is validated against the SSRF deny-list and pinned against DNS
// rebinding, unless allowUntrusted is set. It is the single builder for every
// caller that dials a caller-influenced RemoteConf endpoint: the volume
// FetchAndWriteNeedle write path and the filer and S3 gateway remote-mount read
// paths, which otherwise reach the same s3manager sink unguarded.
func BuildGuardedRemoteStorageClient(ctx context.Context, remoteConf *remote_pb.RemoteConf, allowUntrusted bool) (remote_storage.RemoteStorageClient, error) {
	if !allowUntrusted {
		if remoteConf.GetType() == "gcs" {
			if credsErr := checkGcsCredentials(remoteConf.GetGcsGoogleApplicationCredentials()); credsErr != nil {
				return nil, fmt.Errorf("reject remote credentials: %w", credsErr)
			}
		}
		if _, _, ok := guardedRemoteClient(remoteConf); ok {
			// One client per configuration and credentials version:
			// a client built per call kept its transport's connection open for
			// IdleConnTimeout and never reused it, one open connection per lazy
			// stat, cold read or remote delete. The endpoint is resolved and
			// checked against the deny-list when a client is built (a cache
			// miss); every new connection of a cached client still dials through
			// the guarded dialer, which checks the address it actually connects to.
			buildCtx := context.WithoutCancel(ctx)
			client, err := remote_storage.GetRemoteStorageWith("guarded", remoteConf, guardedCredentialsVersion(remoteConf),
				func(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, func(), error) {
					endpoint, makeClient, ok := guardedRemoteClient(conf)
					if !ok {
						return nil, nil, fmt.Errorf("remote %s: no guarded client", conf.GetName())
					}
					if validateErr := validateRemoteEndpoint(buildCtx, endpoint); validateErr != nil {
						return nil, nil, fmt.Errorf("reject remote endpoint: %w", validateErr)
					}
					// A client whose dial path re-validates the resolved IP every
					// time. This pins the validated endpoint against DNS rebinding
					// (a hostname that resolves to a public IP for
					// validateRemoteEndpoint and then flips to 127.0.0.1 /
					// 169.254.x.x when the SDK dials).
					httpClient := newGuardedHTTPClient(endpoint)
					client, err := makeClient(httpClient)
					if err != nil {
						return nil, nil, fmt.Errorf("get remote client: %w", err)
					}
					return client, httpClient.CloseIdleConnections, nil
				})
			if err != nil {
				return nil, err
			}
			return client, nil
		}
	}
	client, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return nil, fmt.Errorf("get remote client: %w", err)
	}
	return client, nil
}

// ValidateRemoteConfForLoad applies the same SSRF deny-list and gcs credential
// checks BuildGuardedRemoteStorageClient enforces at dial time, but without
// building a client. It is injected into the filer's FilerRemoteStorage so a
// RemoteConf planted under /etc/remote is rejected at load — before the
// lazy-fetch / lazy-list / remote-delete paths can resolve and dial it. A conf
// whose type does not steer a caller-supplied endpoint (and so dials a fixed
// provider host) passes; allowUntrusted skips the check to mirror the volume
// server opt-out.
func ValidateRemoteConfForLoad(ctx context.Context, remoteConf *remote_pb.RemoteConf, allowUntrusted bool) error {
	if remoteConf == nil {
		return nil
	}
	if allowUntrusted {
		return nil
	}
	if remoteConf.GetType() == "gcs" {
		if credsErr := checkGcsCredentials(remoteConf.GetGcsGoogleApplicationCredentials()); credsErr != nil {
			return fmt.Errorf("reject remote credentials: %w", credsErr)
		}
	}
	if endpoint, _, ok := guardedRemoteClient(remoteConf); ok {
		if validateErr := validateRemoteEndpointForLoad(endpoint); validateErr != nil {
			return fmt.Errorf("reject remote endpoint: %w", validateErr)
		}
	}
	return nil
}

// validateRemoteEndpointForLoad applies the static parts of the SSRF deny-list
// (scheme, IMDS hostnames, IP-literal blocked addresses) without resolving
// hostnames. DNS resolution is left to BuildGuardedRemoteStorageClient at dial
// time, so a transient DNS failure during /etc/remote reload cannot drop a
// working mount from the live map.
func validateRemoteEndpointForLoad(endpoint string) error {
	if strings.TrimSpace(endpoint) == "" {
		return fmt.Errorf("remote endpoint is empty")
	}
	u, parseErr := url.Parse(endpoint)
	if parseErr != nil {
		return fmt.Errorf("parse remote endpoint %q: %w", endpoint, parseErr)
	}
	scheme := strings.ToLower(u.Scheme)
	if scheme != "http" && scheme != "https" {
		return fmt.Errorf("remote endpoint %q must use http or https, got %q", endpoint, u.Scheme)
	}
	host := u.Hostname()
	if host == "" {
		return fmt.Errorf("remote endpoint %q has no host", endpoint)
	}
	if _, ok := blockedIMDSHosts[strings.ToLower(host)]; ok {
		return fmt.Errorf("remote endpoint %q targets instance metadata service", endpoint)
	}
	if ip := net.ParseIP(host); ip != nil {
		return checkBlockedIP(endpoint, ip)
	}
	return nil
}

func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_server_pb.FetchAndWriteNeedleRequest) (resp *volume_server_pb.FetchAndWriteNeedleResponse, err error) {
	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return nil, err
	}
	if err := vs.CheckMaintenanceMode(); err != nil {
		return nil, err
	}

	resp = &volume_server_pb.FetchAndWriteNeedleResponse{}
	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	remoteConf := req.RemoteConf

	client, getClientErr := BuildGuardedRemoteStorageClient(ctx, remoteConf, vs.AllowUntrustedRemoteEndpoints)
	if getClientErr != nil {
		return nil, getClientErr
	}

	remoteStorageLocation := req.RemoteLocation

	var data []byte
	var readRemoteErr error
	if cr, ok := client.(remote_storage.RemoteStorageConcurrentReader); ok {
		concurrency := int(req.DownloadConcurrency)
		if concurrency <= 0 {
			concurrency = 0 // let the implementation choose its default
		} else if concurrency > 64 {
			concurrency = 64
		}
		data, readRemoteErr = cr.ReadFileWithConcurrency(remoteStorageLocation, req.Offset, req.Size, concurrency)
	} else {
		data, readRemoteErr = client.ReadFile(remoteStorageLocation, req.Offset, req.Size)
	}
	if readRemoteErr != nil {
		return nil, volumeRemoteReadError(remoteStorageLocation, readRemoteErr)
	}
	// The chunk is recorded with the requested size, so a short read would be
	// cached as a full-size chunk with a zero-padded or truncated tail. Fail
	// loudly instead of persisting silently corrupt content.
	if int64(len(data)) != req.Size {
		return nil, fmt.Errorf("read from remote %+v: got %d bytes, want %d", remoteStorageLocation, len(data), req.Size)
	}

	// Validate every replica target before writing anything, so a malformed or
	// internal target fails the request instead of leaving a local write behind.
	if !vs.AllowUntrustedRemoteEndpoints {
		for _, replica := range req.Replicas {
			if validateErr := validateReplicaTarget(ctx, replica.Url); validateErr != nil {
				return nil, fmt.Errorf("reject replica target: %w", validateErr)
			}
		}
	}

	var wg sync.WaitGroup
	var localErr error
	replicaErrs := make([]error, len(req.Replicas))
	wg.Add(1)
	go func() {
		defer wg.Done()
		n := new(needle.Needle)
		n.Id = types.NeedleId(req.NeedleId)
		n.Cookie = types.Cookie(req.Cookie)
		n.Data, n.DataSize = data, uint32(len(data))
		// copied from *Needle.prepareWriteBuffer()
		n.Size = 4 + types.Size(n.DataSize) + 1
		n.Checksum = needle.NewCRC(n.Data)
		n.LastModified = uint64(time.Now().Unix())
		n.SetHasLastModifiedDate()
		if _, localWriteErr := vs.store.WriteVolumeNeedle(v.Id, n, true, false); localWriteErr != nil {
			localErr = fmt.Errorf("local write needle %d size %d: %v", req.NeedleId, req.Size, localWriteErr)
		} else {
			resp.ETag = n.Etag()
		}
	}()
	if len(req.Replicas) > 0 {
		fileId := needle.NewFileId(v.Id, req.NeedleId, req.Cookie)
		for i, replica := range req.Replicas {
			wg.Add(1)
			go func(idx int, targetVolumeServer string) {
				defer wg.Done()
				uploadUrl, urlErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(
					fmt.Sprintf("%s/%s?type=replicate", targetVolumeServer, fileId.String()))
				if urlErr != nil {
					replicaErrs[idx] = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, urlErr)
					return
				}
				uploadOption := &operation.UploadOption{
					UploadUrl:         uploadUrl,
					Filename:          "",
					Cipher:            false,
					IsInputCompressed: false,
					IsReplication:     true,
					MimeType:          "",
					PairMap:           nil,
					Jwt:               security.EncodedJwt(req.Auth),
				}

				// Upload through a client that re-checks the target at connect
				// time, so a replica hostname cannot rebind to a blocked address
				// after validateReplicaTarget. Peers may be private, so allow
				// private here; the opt-out uses the shared global client.
				var uploader *operation.Uploader
				if vs.AllowUntrustedRemoteEndpoints {
					var uploaderErr error
					uploader, uploaderErr = operation.NewUploader()
					if uploaderErr != nil {
						replicaErrs[idx] = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, uploaderErr)
						return
					}
				} else {
					uploader = operation.NewUploaderWithHttpClient(newGuardedPeerHTTPClient(targetVolumeServer))
				}

				if _, replicaWriteErr := uploader.UploadData(ctx, data, uploadOption); replicaWriteErr != nil {
					replicaErrs[idx] = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, replicaWriteErr)
				}
			}(i, replica.Url)
		}
	}

	wg.Wait()

	// local write error wins; otherwise surface the first replica failure
	err = localErr
	for _, replicaErr := range replicaErrs {
		if err == nil {
			err = replicaErr
		}
	}

	return resp, err
}

// volumeRemoteReadError keeps a confirmed missing object distinguishable across
// gRPC, where a wrapped sentinel would arrive as codes.Unknown text. The
// sentinel's text in the message is what the filer matches, so a NotFound for
// any other reason cannot pass for it.
func volumeRemoteReadError(loc *remote_pb.RemoteStorageLocation, err error) error {
	if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
		return status.Errorf(codes.NotFound, "read from remote %+v: %v", loc, remote_storage.ErrRemoteObjectNotFound)
	}
	return fmt.Errorf("read from remote %+v: %w", loc, err)
}