Files
seaweedfs/weed/server/filer_grpc_server_batch_lookup.go
T
Carlos Leyva 6d0a6e0935 filer: mint read_auth for the data chunks behind manifest chunks (#12756)
* filer: mint read_auth for the data chunks behind manifest chunks

* filer: cover every chunk behind manifests in read_auth (data + nested), minted after resolution, and skip when read signing is off

* filer: pass the master client as manifest cache invalidator, and move the read_auth helpers below populateBatchReadAuth for the enterprise merge
2026-11-18 19:37:06 -08:00

323 lines
12 KiB
Go

package weed_server
import (
"context"
"errors"
"fmt"
"math"
"sort"
"strconv"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
maxFilerBatchLookupRequests = 4096
maxFilerBatchLookupWorkers = 32
)
type filerBatchEntryFinder func(context.Context, util.FullPath) (*filer.Entry, int64, error)
type filerBatchVolumeLookup func(context.Context, []string) (map[string][]wdclient.Location, error)
// LookupDirectoryEntries performs exact entry lookups concurrently while
// preserving request order, then resolves every referenced volume in one
// deduplicated master lookup.
func (fs *FilerServer) LookupDirectoryEntries(ctx context.Context, req *filer_pb.LookupDirectoryEntriesRequest) (*filer_pb.LookupDirectoryEntriesResponse, error) {
lookupVolumes := selectFilerBatchVolumeLookup(req,
fs.filer.MasterClient.LookupVolumeIdsWithFallback,
fs.filer.MasterClient.LookupVolumeIdsAuthoritative)
response, err := lookupDirectoryEntries(ctx, req, fs.filer.Signature, fs.fencedFindEntry, lookupVolumes)
if err != nil {
return nil, err
}
if fs.volumeGuard != nil {
readSigningEnabled := len(fs.volumeGuard.ReadSigningKey()) > 0
populateReadAuth(ctx, response, readSigningEnabled, fs.filer.MasterClient.GetLookupFileIdFunction(), fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken)
}
return response, nil
}
func selectFilerBatchVolumeLookup(req *filer_pb.LookupDirectoryEntriesRequest, cached, authoritative filerBatchVolumeLookup) filerBatchVolumeLookup {
if req != nil && req.GetUnavailableVolumeIsMiss() {
// DeletedVids is an asynchronous cache invalidation hint and its delivery
// is deliberately best-effort. Cache reads must therefore confirm the
// unique Volume IDs with Master before treating a location as live.
return authoritative
}
return cached
}
func populateBatchReadAuth(response *filer_pb.LookupDirectoryEntriesResponse, mint func(string) string) {
response.ReadAuth = make(map[string]string)
for _, result := range response.Results {
if result == nil || result.Entry == nil {
continue
}
for _, chunk := range result.Entry.Chunks {
fid := chunk.GetFileIdString()
if fid == "" {
continue
}
if token := mint(fid); token != "" {
response.ReadAuth[fid] = token
}
}
}
}
// populateReadAuth mints read_auth tokens for every chunk a reader must fetch:
// the chunks listed on each entry, plus the chunks behind any manifest chunks --
// the data chunks and any intermediate (nested) manifest chunks, since the entry
// lists only the top-level manifests and a client reading with filer-issued
// tokens could fetch those but not the chunks they resolve to.
//
// It resolves manifests first and mints every token afterwards. Read JWTs are
// short-lived (default 60s), so a token must not be signed before the manifest
// downloads this lookup needs have run: otherwise a token minted early -- for a
// listed chunk, an ordinary file, or an earlier entry's chunks -- could expire
// while later entries' manifests download, and the response would carry an
// already-expired token.
func populateReadAuth(ctx context.Context, response *filer_pb.LookupDirectoryEntriesResponse, readSigningEnabled bool, lookupFileIdFn wdclient.LookupFileIdFunctionType, invalidator filer.CacheInvalidator, mint func(string) string) {
manifestBackedFids := resolveFidsBehindManifests(ctx, response, readSigningEnabled, lookupFileIdFn, invalidator)
populateBatchReadAuth(response, mint)
mintReadAuth(response, manifestBackedFids, mint)
}
// resolveFidsBehindManifests returns the fids a reader must fetch to read the
// files behind the response's manifest chunks: the leaf data chunks and every
// intermediate (nested) manifest chunk ResolveChunkManifest traverses to reach
// them. The top-level manifest chunks are listed on the entry and already minted
// by populateBatchReadAuth; the dedup in mintReadAuth drops that overlap, so the
// net new tokens are the data chunks and any nested manifests.
//
// It resolves manifests -- which downloads and expands them from the volume
// servers -- only when read signing is enabled: with no read key the volume
// server does not enforce read JWTs and mint returns "" for every chunk, so the
// download would be wasted I/O that yields no token. The invalidator (the master
// client, as the filer's own resolveManifest read path passes) lets a manifest
// whose volume moved refresh the stale cached location and retry instead of
// failing and leaving its data chunks without tokens.
func resolveFidsBehindManifests(ctx context.Context, response *filer_pb.LookupDirectoryEntriesResponse, readSigningEnabled bool, lookupFileIdFn wdclient.LookupFileIdFunctionType, invalidator filer.CacheInvalidator) []string {
if !readSigningEnabled {
return nil
}
var fids []string
for _, result := range response.Results {
if result == nil || result.Entry == nil || !filer.HasChunkManifest(result.Entry.Chunks) {
continue
}
dataChunks, manifestChunks, err := filer.ResolveChunkManifest(ctx, lookupFileIdFn, result.Entry.Chunks, 0, math.MaxInt64, invalidator)
if err != nil {
glog.V(1).InfofCtx(ctx, "read auth: resolve manifests of %s: %v", result.Entry.Name, err)
continue
}
for _, chunk := range dataChunks {
if fid := chunk.GetFileIdString(); fid != "" {
fids = append(fids, fid)
}
}
for _, chunk := range manifestChunks {
if fid := chunk.GetFileIdString(); fid != "" {
fids = append(fids, fid)
}
}
}
return fids
}
// mintReadAuth adds a read_auth token for each fid not already covered.
func mintReadAuth(response *filer_pb.LookupDirectoryEntriesResponse, fids []string, mint func(string) string) {
for _, fid := range fids {
if response.ReadAuth[fid] != "" {
continue
}
if token := mint(fid); token != "" {
response.ReadAuth[fid] = token
}
}
}
func lookupDirectoryEntries(
ctx context.Context,
req *filer_pb.LookupDirectoryEntriesRequest,
logSignature int32,
findEntry filerBatchEntryFinder,
lookupVolumes filerBatchVolumeLookup,
) (*filer_pb.LookupDirectoryEntriesResponse, error) {
if err := validateFilerBatchLookupRequest(req); err != nil {
return nil, err
}
results := make([]*filer_pb.LookupDirectoryEntryResult, len(req.Requests))
resultVolumeIDs := make([]map[string]struct{}, len(req.Requests))
jobs := make(chan int)
workerCount := min(len(req.Requests), maxFilerBatchLookupWorkers)
var workers sync.WaitGroup
workers.Add(workerCount)
for range workerCount {
go func() {
defer workers.Done()
for index := range jobs {
request := req.Requests[index]
entry, logTsNs, err := findEntry(ctx, util.JoinPath(request.Directory, request.Name))
result := &filer_pb.LookupDirectoryEntryResult{
LogTsNs: logTsNs,
LogSignature: logSignature,
}
switch {
case errors.Is(err, filer_pb.ErrNotFound):
results[index] = result
case err != nil:
result.Error = err.Error()
results[index] = result
case entry == nil:
result.Error = "entry lookup returned no entry"
results[index] = result
default:
pbEntry := entry.ToProtoEntry()
volumeIDs, volumeErr := filerBatchEntryVolumeIDs(pbEntry)
result.Found = true
result.Entry = pbEntry
if volumeErr != nil {
result.Error = volumeErr.Error()
}
results[index] = result
resultVolumeIDs[index] = volumeIDs
}
}
}()
}
for index := range req.Requests {
select {
case jobs <- index:
case <-ctx.Done():
close(jobs)
workers.Wait()
return nil, status.FromContextError(ctx.Err()).Err()
}
}
close(jobs)
workers.Wait()
if err := ctx.Err(); err != nil {
return nil, status.FromContextError(err).Err()
}
allVolumeIDs := make(map[string]struct{})
for _, volumeIDs := range resultVolumeIDs {
for volumeID := range volumeIDs {
allVolumeIDs[volumeID] = struct{}{}
}
}
volumeIDs := make([]string, 0, len(allVolumeIDs))
for volumeID := range allVolumeIDs {
volumeIDs = append(volumeIDs, volumeID)
}
sort.Strings(volumeIDs)
response := &filer_pb.LookupDirectoryEntriesResponse{
Results: results,
LocationsMap: make(map[string]*filer_pb.Locations, len(volumeIDs)),
}
if len(volumeIDs) == 0 {
return response, nil
}
locationsByVolume, lookupErr := lookupVolumes(ctx, volumeIDs)
// The provider returns a nil map when it got no answer, and a populated map
// with the volumes the master does not serve reported as errors when it did,
// so a nil map is the only sign of an unanswered lookup. Only a volume the
// master itself left out may become a miss; anything else stays an error.
missIsAuthoritative := req.UnavailableVolumeIsMiss && locationsByVolume != nil
for _, volumeID := range volumeIDs {
locations := locationsByVolume[volumeID]
if len(locations) != 0 {
response.LocationsMap[volumeID] = &filer_pb.Locations{
Locations: wdclientLocationsToPb(locations),
}
continue
}
if !missIsAuthoritative {
response.LocationsMap[volumeID] = &filer_pb.Locations{}
}
for index, entryVolumeIDs := range resultVolumeIDs {
if _, affected := entryVolumeIDs[volumeID]; !affected {
continue
}
if missIsAuthoritative && results[index].Error == "" {
results[index].Found = false
results[index].Entry = nil
resultVolumeIDs[index] = nil
continue
}
message := fmt.Sprintf("volume %s has no locations", volumeID)
if lookupErr != nil {
message += ": " + lookupErr.Error()
}
results[index].Error = appendFilerBatchLookupError(results[index].Error, message)
}
}
return response, nil
}
func validateFilerBatchLookupRequest(req *filer_pb.LookupDirectoryEntriesRequest) error {
if req == nil || len(req.Requests) == 0 {
return status.Error(codes.InvalidArgument, "batch lookup requires at least one request")
}
if len(req.Requests) > maxFilerBatchLookupRequests {
return status.Errorf(codes.ResourceExhausted, "batch lookup has %d requests; maximum is %d", len(req.Requests), maxFilerBatchLookupRequests)
}
for index, request := range req.Requests {
if request == nil {
return status.Errorf(codes.InvalidArgument, "batch lookup request %d is missing", index)
}
if request.Directory == "" || !strings.HasPrefix(request.Directory, "/") || strings.ContainsRune(request.Directory, '\x00') {
return status.Errorf(codes.InvalidArgument, "batch lookup request %d has invalid directory", index)
}
if request.Name == "" || request.Name == "." || request.Name == ".." || strings.ContainsAny(request.Name, "/\x00") {
return status.Errorf(codes.InvalidArgument, "batch lookup request %d has invalid name", index)
}
}
return nil
}
func filerBatchEntryVolumeIDs(entry *filer_pb.Entry) (map[string]struct{}, error) {
volumeIDs := make(map[string]struct{})
for index, chunk := range entry.GetChunks() {
if chunk == nil {
return volumeIDs, fmt.Errorf("entry chunk %d is missing", index)
}
if chunk.Fid != nil {
volumeIDs[strconv.FormatUint(uint64(chunk.Fid.VolumeId), 10)] = struct{}{}
continue
}
if chunk.FileId == "" {
return volumeIDs, fmt.Errorf("entry chunk %d has no file id", index)
}
fid, err := needle.ParseFileIdFromString(chunk.FileId)
if err != nil {
return volumeIDs, fmt.Errorf("entry chunk %d has invalid file id: %w", index, err)
}
volumeIDs[strconv.FormatUint(uint64(fid.VolumeId), 10)] = struct{}{}
}
return volumeIDs, nil
}
func appendFilerBatchLookupError(existing, next string) string {
if existing == "" {
return next
}
return existing + "; " + next
}