read cold remote objects straight from the origin while caching (#10731)

* refactor: extract remote mount resolution into shared helpers

* refactor: share the adaptive remote cache wait policy

* filer: stream cold remote reads from the origin while caching

* s3: stream cold remote reads from the origin instead of 503 retries

* test: cover the S3 origin stream-through path

* remote mounts: match on path components and prefer the longest mount

* fail short origin streams instead of silently truncating

* s3: try the origin before failing a cold read on a local cache error

* s3: gate origin streaming on the entry's resolved version

* return the cache RPC's NotFound as a canonical status and classify it everywhere

* filer: keep multipart-range cold reads on the retry path

weed/server/filer_grpc_server_remote.go

@@ -2,9 +2,9 @@ package weed_server
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sort"
-	"strings"
 	"sync"
 	"time"
 
@@ -17,6 +17,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -48,6 +50,11 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
 			glog.V(2).Infof("CacheRemoteObjectToLocalCluster: shared result for %s", cacheKey)
 		}
 		if res.Err != nil {
+			// The sentinel would cross gRPC as codes.Unknown; make it canonical
+			// so remote callers can classify a vanished entry.
+			if errors.Is(res.Err, filer_pb.ErrNotFound) {
+				return nil, status.Error(codes.NotFound, res.Err.Error())
+			}
 			return nil, res.Err
 		}
 		if res.Val == nil {
@@ -90,37 +97,10 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re
 
 	glog.V(1).Infof("CacheRemoteObjectToLocalCluster: caching %s/%s (remote size: %d)", req.Directory, req.Name, entry.Remote.RemoteSize)
 
-	// load all mappings
-	mappingEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE))
+	storageConf, remoteLocation, err := fs.resolveMountedRemote(ctx, req.Directory, req.Name)
 	if err != nil {
 		return nil, err
 	}
-	mappings, err := filer.UnmarshalRemoteStorageMappings(mappingEntry.Content)
-	if err != nil {
-		return nil, err
-	}
-
-	// find mapping
-	var remoteStorageMountedLocation *remote_pb.RemoteStorageLocation
-	var localMountedDir string
-	for k, loc := range mappings.Mappings {
-		if strings.HasPrefix(req.Directory, k) {
-			localMountedDir, remoteStorageMountedLocation = k, loc
-		}
-	}
-	if localMountedDir == "" {
-		return nil, fmt.Errorf("%s is not mounted", req.Directory)
-	}
-
-	// find storage configuration
-	storageConfEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, remoteStorageMountedLocation.Name+filer.REMOTE_STORAGE_CONF_SUFFIX))
-	if err != nil {
-		return nil, err
-	}
-	storageConf := &remote_pb.RemoteConf{}
-	if unMarshalErr := proto.Unmarshal(storageConfEntry.Content, storageConf); unMarshalErr != nil {
-		return nil, fmt.Errorf("unmarshal remote storage conf %s/%s: %v", filer.DirectoryEtcRemote, remoteStorageMountedLocation.Name+filer.REMOTE_STORAGE_CONF_SUFFIX, unMarshalErr)
-	}
 
 	// detect storage option
 	so, err := fs.detectStorageOption(ctx, req.Directory, "", "", 0, "", "", "", "")
@@ -156,8 +136,6 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re
 		altRequest.ExpectedDataSize = uint64(chunkSize)
 	}
 
-	dest := util.FullPath(remoteStorageMountedLocation.Path).Child(string(util.FullPath(req.Directory).Child(req.Name))[len(localMountedDir):])
-
 	var chunks []*filer_pb.FileChunk
 	var chunksMu sync.Mutex
 	var fetchAndWriteErr error
@@ -239,14 +217,10 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re
 					Auth:                string(assignResult.Auth),
 					DownloadConcurrency: downloadConcurrency,
 					RemoteConf:          storageConf,
-					RemoteLocation: &remote_pb.RemoteStorageLocation{
-						Name:   remoteStorageMountedLocation.Name,
-						Bucket: remoteStorageMountedLocation.Bucket,
-						Path:   string(dest),
-					},
+					RemoteLocation:      remoteLocation,
 				})
 				if fetchErr != nil {
-					return fmt.Errorf("volume server %s fetchAndWrite %s: %v", assignResult.Url, dest, fetchErr)
+					return fmt.Errorf("volume server %s fetchAndWrite %s: %v", assignResult.Url, remoteLocation.Path, fetchErr)
 				}
 				etag = resp.ETag
 				return nil
@@ -352,3 +326,33 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re
 	return resp, nil
 
 }
+
+// resolveMountedRemote reads /etc/remote fresh (so conf changes need no restart)
+// and maps dir/name to its remote storage conf and remote location.
+func (fs *FilerServer) resolveMountedRemote(ctx context.Context, dir, name string) (*remote_pb.RemoteConf, *remote_pb.RemoteStorageLocation, error) {
+	mappingEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE))
+	if err != nil {
+		return nil, nil, err
+	}
+	mappings, err := filer.UnmarshalRemoteStorageMappings(mappingEntry.Content)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	localMountedDir, remoteStorageMountedLocation, err := filer.FindMountedRemoteMapping(mappings, dir)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	storageConfEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, remoteStorageMountedLocation.Name+filer.REMOTE_STORAGE_CONF_SUFFIX))
+	if err != nil {
+		return nil, nil, err
+	}
+	storageConf := &remote_pb.RemoteConf{}
+	if unMarshalErr := proto.Unmarshal(storageConfEntry.Content, storageConf); unMarshalErr != nil {
+		return nil, nil, fmt.Errorf("unmarshal remote storage conf %s/%s: %v", filer.DirectoryEtcRemote, remoteStorageMountedLocation.Name+filer.REMOTE_STORAGE_CONF_SUFFIX, unMarshalErr)
+	}
+
+	remoteLocation := filer.MapFullPathToRemoteStorageLocation(util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(dir).Child(name))
+	return storageConf, remoteLocation, nil
+}

weed/server/filer_server_handlers_read.go

@@ -16,10 +16,13 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // Validates the preconditions. Returns true if GET/HEAD operation should not proceed.
@@ -213,26 +216,43 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		chunks := entry.GetChunks()
 		if entry.IsInRemoteOnly() {
 			dir, name := entry.FullPath.DirAndName()
-			if resp, err := fs.CacheRemoteObjectToLocalCluster(ctx, &filer_pb.CacheRemoteObjectToLocalClusterRequest{
+			// Bounded wait: a large download outlasts any client timeout, so
+			// serve straight from the origin once the wait expires while the
+			// detached cache keeps filling for later reads.
+			cacheCtx, cancelCache := context.WithTimeout(ctx, remote_storage.CacheWaitTimeout(entry.Remote.RemoteSize))
+			resp, err := fs.CacheRemoteObjectToLocalCluster(cacheCtx, &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 				Directory: dir,
 				Name:      name,
-			}); err != nil {
+			})
+			cancelCache()
+			if err != nil {
 				stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadCache).Inc()
 				// Client disconnected: surface ctx error so caller stays silent.
 				if ctxErr := ctx.Err(); ctxErr != nil {
 					return nil, ctxErr
 				}
-				// Entry vanished mid-cache: forward NotFound so caller maps to 404,
-				// not the 503 retry-loop.
-				if errors.Is(err, filer_pb.ErrNotFound) {
-					return nil, err
+				// Entry vanished mid-cache: forward the sentinel so caller maps to
+				// 404, not the 503 retry-loop. The cache RPC returns it as a
+				// canonical status, which errors.Is cannot see.
+				if errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound {
+					return nil, filer_pb.ErrNotFound
 				}
-				// Cache still filling: tag with sentinel so caller maps to 503 + Retry-After.
+				// A multipart Range prepares every part before writing any, which
+				// would hold one open origin connection per part and leak them
+				// when a later prepare fails; keep those on the 503 retry path.
+				if !strings.Contains(r.Header.Get("Range"), ",") {
+					glog.V(1).InfofCtx(ctx, "stream %s from remote while caching: %v", entry.FullPath, err)
+					if streamFn, remoteErr := fs.streamFromRemote(ctx, dir, name, offset, size); remoteErr == nil {
+						return streamFn, nil
+					} else {
+						glog.WarningfCtx(ctx, "stream %s from remote: %v", entry.FullPath, remoteErr)
+					}
+				}
+				// Origin unreadable: tag with sentinel so caller maps to 503 + Retry-After.
 				glog.WarningfCtx(ctx, "CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
 				return nil, fmt.Errorf("cache %s: %w", entry.FullPath, ErrCacheNotReady)
-			} else {
-				chunks = resp.Entry.GetChunks()
 			}
+			chunks = resp.Entry.GetChunks()
 		}
 
 		// Use a detached context for streaming so client disconnects/cancellations don't abort volume server operations,
@@ -259,6 +279,53 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 	})
 }
 
+// streamFromRemote serves a byte range of a remote-only entry straight from the
+// mounted origin, so a first read is not blocked by the full local caching.
+func (fs *FilerServer) streamFromRemote(ctx context.Context, dir, name string, offset, size int64) (filer.DoStreamContent, error) {
+	storageConf, remoteLocation, err := fs.resolveMountedRemote(ctx, dir, name)
+	if err != nil {
+		return nil, err
+	}
+	client, err := remote_storage.GetRemoteStorage(storageConf)
+	if err != nil {
+		return nil, err
+	}
+	streamer, ok := client.(remote_storage.RemoteStorageStreamReader)
+	if !ok {
+		return nil, fmt.Errorf("remote storage type %s does not support streaming reads", storageConf.Type)
+	}
+	reader, err := streamer.ReadFileAsStream(ctx, remoteLocation, offset, size)
+	if err != nil {
+		return nil, err
+	}
+	downloadThrottler := util.NewWriteThrottler(fs.option.DownloadMaxBytesPs)
+	return func(writer io.Writer) error {
+		defer reader.Close()
+		var written int64
+		buf := make([]byte, 128*1024)
+		for {
+			n, readErr := reader.Read(buf)
+			if n > 0 {
+				if _, writeErr := writer.Write(buf[:n]); writeErr != nil {
+					return writeErr
+				}
+				written += int64(n)
+				downloadThrottler.MaybeSlowdown(int64(n))
+			}
+			if readErr == io.EOF {
+				if written != size {
+					// the origin returned fewer bytes than the entry's RemoteSize
+					return fmt.Errorf("origin stream %s: %w after %d of %d bytes", remoteLocation.Path, io.ErrUnexpectedEOF, written, size)
+				}
+				return nil
+			}
+			if readErr != nil {
+				return readErr
+			}
+		}
+	}, nil
+}
+
 func (fs *FilerServer) maybeGetVolumeReadJwtAuthorizationToken(fileId string) string {
 	// Only ever sign with the read key. A volume server enforces read JWTs
 	// solely when jwt.signing.read.key is set, so falling back to the write key