Merge branch 'master' of https://github.com/seaweedfs/seaweedfs into nfs-enable-inode-index

weed/admin/handlers/file_browser_grpc.go

@@ -153,6 +153,9 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
 		ChunkSize:  adminUploadChunkSize,
 		MimeType:   mimeType,
 		AssignFunc: assignFunc,
+		LookupJwtFunc: func(lookupCtx context.Context, fileId string) security.EncodedJwt {
+			return operation.LookupJwt(lookupCtx, h.adminServer.GetMasterClient().GetMaster(lookupCtx), h.adminServer.GetGrpcDialOption(), fileId)
+		},
 	})
 	if err != nil {
 		// Partial chunks come back even on error so we can clean them up rather

weed/admin/plugin/plugin_scheduler.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
@@ -614,7 +615,7 @@ func deriveSchedulerAdminRuntime(
 	descriptor *plugin_pb.JobTypeDescriptor,
 ) *plugin_pb.AdminRuntimeConfig {
 	if cfg != nil && cfg.AdminRuntime != nil {
-		adminConfig := *cfg.AdminRuntime
+		adminConfig := proto.Clone(cfg.AdminRuntime).(*plugin_pb.AdminRuntimeConfig)
 		// Overlay descriptor defaults for any zero numeric fields. Persisted
 		// configs from older versions have no execution_timeout_seconds, and
 		// without this overlay the scheduler would fall back to the 90s
@@ -646,7 +647,7 @@ func deriveSchedulerAdminRuntime(
 				adminConfig.ExecutionTimeoutSeconds = defaults.ExecutionTimeoutSeconds
 			}
 		}
-		return &adminConfig
+		return adminConfig
 	}
 
 	if descriptor == nil || descriptor.AdminRuntimeDefaults == nil {

weed/admin/plugin/registry.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/protobuf/proto"
 )
 
 const defaultWorkerStaleTimeout = 2 * time.Minute
@@ -495,36 +496,35 @@ func cloneJobTypeCapability(in *plugin_pb.JobTypeCapability) *plugin_pb.JobTypeC
 	if in == nil {
 		return nil
 	}
-	out := *in
-	return &out
+	return proto.Clone(in).(*plugin_pb.JobTypeCapability)
 }
 
 func cloneWorkerHeartbeat(in *plugin_pb.WorkerHeartbeat) *plugin_pb.WorkerHeartbeat {
 	if in == nil {
 		return nil
 	}
-	out := *in
+	out := proto.Clone(in).(*plugin_pb.WorkerHeartbeat)
 	if in.RunningWork != nil {
-		out.RunningWork = make([]*plugin_pb.RunningWork, 0, len(in.RunningWork))
-		for _, rw := range in.RunningWork {
-			if rw == nil {
-				continue
+		running := out.RunningWork[:0]
+		// Clone can turn nil messages into empty messages, so filter using
+		// their original positions while reusing the cloned entries.
+		for i, work := range in.RunningWork {
+			if work != nil {
+				running = append(running, out.RunningWork[i])
 			}
-			clone := *rw
-			out.RunningWork = append(out.RunningWork, &clone)
 		}
-	}
-	if in.QueuedJobsByType != nil {
-		out.QueuedJobsByType = make(map[string]int32, len(in.QueuedJobsByType))
-		for k, v := range in.QueuedJobsByType {
-			out.QueuedJobsByType[k] = v
+		clear(out.RunningWork[len(running):])
+		if running == nil {
+			running = []*plugin_pb.RunningWork{}
 		}
+		out.RunningWork = running
 	}
-	if in.Metadata != nil {
-		out.Metadata = make(map[string]string, len(in.Metadata))
-		for k, v := range in.Metadata {
-			out.Metadata[k] = v
-		}
+	// Preserve writable empty maps without rebuilding populated clones.
+	if in.QueuedJobsByType != nil && out.QueuedJobsByType == nil {
+		out.QueuedJobsByType = make(map[string]int32)
 	}
-	return &out
+	if in.Metadata != nil && out.Metadata == nil {
+		out.Metadata = make(map[string]string)
+	}
+	return out
 }

weed/admin/plugin/registry_snapshot_test.go

@@ -0,0 +1,65 @@
+package plugin
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+func TestCloneWorkerHeartbeatFiltersNilAndIsolatesState(t *testing.T) {
+	t.Parallel()
+	source := &plugin_pb.WorkerHeartbeat{
+		RunningWork:      []*plugin_pb.RunningWork{nil, {WorkId: "first"}, nil, {WorkId: "second"}, nil},
+		QueuedJobsByType: map[string]int32{"vacuum": 2},
+		Metadata:         map[string]string{"rack": "rack-a"},
+	}
+	got := cloneWorkerHeartbeat(source)
+	if len(got.RunningWork) != 2 || got.RunningWork[0].GetWorkId() != "first" || got.RunningWork[1].GetWorkId() != "second" {
+		t.Fatalf("unexpected filtered work: %v", got.RunningWork)
+	}
+	got.RunningWork[0].WorkId = "changed"
+	got.QueuedJobsByType["vacuum"] = 99
+	got.Metadata["rack"] = "changed"
+	if len(source.RunningWork) != 5 || source.RunningWork[0] != nil || source.RunningWork[1].WorkId != "first" {
+		t.Fatalf("cloning mutated source work: %v", source.RunningWork)
+	}
+	if source.QueuedJobsByType["vacuum"] != 2 || source.Metadata["rack"] != "rack-a" {
+		t.Fatal("cloned maps alias the source")
+	}
+}
+
+func TestCloneWorkerHeartbeatPreservesEmptyCollections(t *testing.T) {
+	t.Parallel()
+	if cloneWorkerHeartbeat(nil) != nil {
+		t.Fatal("nil heartbeat must stay nil")
+	}
+	for _, tc := range []struct {
+		name        string
+		source      *plugin_pb.WorkerHeartbeat
+		wantNilWork bool
+	}{
+		{"nil", &plugin_pb.WorkerHeartbeat{}, true},
+		{"empty", &plugin_pb.WorkerHeartbeat{
+			RunningWork:      []*plugin_pb.RunningWork{},
+			QueuedJobsByType: map[string]int32{},
+			Metadata:         map[string]string{},
+		}, false},
+		{"all nil work", &plugin_pb.WorkerHeartbeat{RunningWork: []*plugin_pb.RunningWork{nil, nil}}, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got := cloneWorkerHeartbeat(tc.source)
+			if len(got.RunningWork) != 0 || (got.RunningWork == nil) != tc.wantNilWork {
+				t.Fatalf("unexpected empty work slice: %#v", got.RunningWork)
+			}
+			if (got.QueuedJobsByType == nil) != (tc.source.QueuedJobsByType == nil) || (got.Metadata == nil) != (tc.source.Metadata == nil) {
+				t.Fatal("clone changed whether empty maps are writable")
+			}
+			if got.Metadata != nil {
+				got.Metadata["added"] = "value"
+				if len(tc.source.Metadata) != 0 {
+					t.Fatal("empty metadata map aliases source")
+				}
+			}
+		})
+	}
+}

weed/cluster/lock_ring_manager.go

@@ -7,6 +7,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/protobuf/proto"
 )
 
 const LockRingStabilizationInterval = 1 * time.Second
@@ -102,9 +103,7 @@ func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.
 	if !ok || update == nil {
 		return nil
 	}
-	cp := *update
-	cp.Servers = append([]string(nil), update.Servers...)
-	return &cp
+	return proto.Clone(update).(*master_pb.LockRingUpdate)
 }
 
 // scheduleBroadcast resets the stabilization timer. If another change arrives

weed/command/benchmark.go

@@ -238,7 +238,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				}
 				var jwtAuthorization security.EncodedJwt
 				if isSecure {
-					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid)
+					jwtAuthorization = operation.LookupJwt(context.Background(), b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid)
 				}
 				if e := util_http.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++

weed/command/filer.go

@@ -51,6 +51,15 @@ var (
 // S3 gateway, whose remote-mount read paths dial the mounted endpoint directly.
 const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only entry accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints."
 
+// iamSelfServiceRequiresGrantUsage documents the flag shared by every command
+// that runs the embedded IAM API.
+const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity."
+
+// allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal
+// object-key character instead of normalizing it to '/'. Repeated '//' in a
+// key still collapses: a filer entry name cannot contain an empty segment.
+const allowBackslashInKeyUsage = "treat backslash as a regular object-key character instead of converting it to '/' (aws-compatible; '//' still collapses)"
+
 type FilerOptions struct {
 	masters                   *pb.ServerDiscovery
 	mastersString             *string
@@ -173,6 +182,8 @@ func init() {
 	filerS3Options.cacheSizeMB = cmdFiler.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
 	filerS3Options.readerCacheSizeMB = cmdFiler.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
 	filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
+	filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -191,6 +202,7 @@ func init() {
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
 	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
 	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
+	filerIamOptions.selfServiceRequiresGrant = filerS3Options.iamSelfServiceRequiresGrant
 
 	filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server")
 	filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port")

weed/command/filer_remote_gateway_buckets.go

@@ -214,11 +214,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 		}
 		if filer_pb.IsCreate(resp) {
 			if message.NewParentPath == option.bucketsDir {
+				if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+					return nil
+				}
 				return handleCreateBucket(message.NewEntry)
 			}
 			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			// Propagate delete markers as deletions on the remote.
 			// Delete markers are zero-content version entries, so they
 			// would be filtered out by the HasData check below.
@@ -292,6 +298,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				return nil
 			}
 			if resp.Directory == option.bucketsDir {
+				if isSystemPath(resp.Directory, message.OldEntry.Name) {
+					return nil
+				}
 				return handleDeleteBucket(message.OldEntry)
 			}
 			// Skip deletion of internal version files; individual version
@@ -300,6 +309,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
 				return nil
 			}
+			if isSystemPath(resp.Directory, message.OldEntry.Name) {
+				return nil
+			}
 			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
 			if !ok {
 				return nil
@@ -323,6 +335,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 					if message.OldEntry.Name == message.NewEntry.Name {
 						return nil
 					}
+					if isSystemPath(resp.Directory, message.OldEntry.Name) || isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+						return nil
+					}
 					if err := handleCreateBucket(message.NewEntry); err != nil {
 						return err
 					}
@@ -339,6 +354,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
 			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
 			if oldOk && newOk {

weed/command/filer_remote_sync_dir.go

@@ -144,6 +144,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
+				return nil
+			}
 			// Propagate delete markers as deletions on the remote.
 			// Delete markers are zero-content version entries, so they
 			// would be filtered out by the HasData check below.
@@ -229,6 +233,10 @@ func processDeleteEvent(
 		glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
 		return nil
 	}
+	if isSystemPath(resp.Directory, message.OldEntry.Name) {
+		glog.V(2).Infof("skipping delete of system path: %s/%s", resp.Directory, message.OldEntry.Name)
+		return nil
+	}
 	if filer.IsMetadataOnlyDelete(message.OldEntry) {
 		glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
 		return nil
@@ -339,6 +347,10 @@ func processUpdateEvent(
 		glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
 		return nil
 	}
+	if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+		glog.V(2).Infof("skipping update of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
+		return nil
+	}
 	oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 	dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 	eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()}
@@ -428,7 +440,7 @@ func processUpdateEvent(
 		}
 		glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
 		if err := client.DeleteFile(oldDest); err != nil {
-			if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) {
+			if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) || isSystemPath(resp.Directory, message.OldEntry.Name) {
 				return nil
 			}
 			if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
@@ -881,6 +893,23 @@ func isMultipartUploadDir(dir string) bool {
 		strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/")
 }
 
+// isSystemPath returns true if the dir and name refer to the internal .system folder
+// under /buckets (such as the bucket-owner index at /buckets/.system/owners/...)
+// or any entry inside it. These paths are internal to SeaweedFS and must not
+// be synced to remote storage.
+func isSystemPath(dir string, name string) bool {
+	fullPath := string(util.NewFullPath(dir, name))
+	trimmed := strings.Trim(fullPath, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) >= 1 && parts[0] == s3_constants.SystemFolder {
+		return true
+	}
+	if len(parts) >= 2 && parts[1] == s3_constants.SystemFolder {
+		return true
+	}
+	return false
+}
+
 // isDeleteMarker returns true if the entry is an S3 delete marker
 // (a zero-content version entry with ExtDeleteMarkerKey set to "true").
 func isDeleteMarker(entry *filer_pb.Entry) bool {

weed/command/filer_remote_sync_dir_test.go

@@ -2016,3 +2016,74 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) {
 		t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry)
 	}
 }
+
+func TestIsSystemPath(t *testing.T) {
+	tests := []struct {
+		dir      string
+		name     string
+		expected bool
+	}{
+		{dir: "/buckets", name: ".system", expected: true},
+		{dir: "/buckets/.system", name: "owners", expected: true},
+		{dir: "/buckets/.system/owners", name: "alice", expected: true},
+		{dir: "/buckets/.system/owners/alice", name: "mybucket", expected: true},
+		{dir: "/", name: ".system", expected: true},
+		{dir: "/.system", name: "config", expected: true},
+		{dir: "/custom-mount", name: ".system", expected: true},
+		{dir: "/custom-mount/.system/owners", name: "alice", expected: true},
+		// Bucket contents should not match
+		{dir: "/buckets/mybucket", name: "file.txt", expected: false},
+		{dir: "/buckets/mybucket", name: ".system", expected: false},
+		{dir: "/buckets/mybucket/.system", name: "data", expected: false},
+		{dir: "/buckets", name: ".system-other", expected: false},
+		{dir: "/buckets/.system-other", name: "sub", expected: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s/%s", tt.dir, tt.name), func(t *testing.T) {
+			got := isSystemPath(tt.dir, tt.name)
+			if got != tt.expected {
+				t.Errorf("isSystemPath(%q, %q) = %v, want %v", tt.dir, tt.name, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestRemoteSyncSkipsSystemFolder(t *testing.T) {
+	const mountedDir = "/buckets"
+	mountLoc := &remote_pb.RemoteStorageLocation{Name: "azure", Bucket: "project-storage", Path: "/"}
+
+	// Test update event on /buckets/.system/owners/... is skipped
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/.system/owners/structify-api",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "structify-copy-canary"},
+			NewParentPath: "/buckets/.system/owners/structify-api",
+			NewEntry:      &filer_pb.Entry{Name: "structify-copy-canary"},
+		},
+	}
+	remote := &recordingRemote{}
+	filerClient := &stubFilerClient{}
+	if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, updateResp); err != nil {
+		t.Fatalf("processUpdateEvent failed: %v", err)
+	}
+	if len(remote.writes) != 0 || len(remote.metadataUpdates) != 0 {
+		t.Fatalf("expected 0 remote writes or metadata updates for system path, got %d writes and %d metadata updates",
+			len(remote.writes), len(remote.metadataUpdates))
+	}
+
+	// Test delete event on /buckets/.system/owners/... is skipped
+	deleteResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/.system/owners/structify-api",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"},
+		},
+	}
+	if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleteResp); err != nil {
+		t.Fatalf("processDeleteEvent failed: %v", err)
+	}
+	if len(remote.deletes) != 0 {
+		t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes))
+	}
+}
+

weed/command/iam.go

@@ -28,10 +28,11 @@ var (
 )
 
 type IamOptions struct {
-	filer   *string
-	masters *string
-	ip      *string
-	port    *int
+	filer                    *string
+	masters                  *string
+	ip                       *string
+	port                     *int
+	selfServiceRequiresGrant *bool
 }
 
 func init() {
@@ -40,6 +41,7 @@ func init() {
 	iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
 	iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
+	iamStandaloneOptions.selfServiceRequiresGrant = cmdIam.Flag.Bool("selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
 }
 
 var cmdIam = &Command{
@@ -101,10 +103,11 @@ func (iamopt *IamOptions) startIamServer() bool {
 	masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
 	router := mux.NewRouter().SkipClean(true)
 	iamApiServer, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
-		Masters:        masters,
-		Filers:         filerAddresses,
-		Port:           *iamopt.port,
-		GrpcDialOption: grpcDialOption,
+		Masters:                     masters,
+		Filers:                      filerAddresses,
+		Port:                        *iamopt.port,
+		GrpcDialOption:              grpcDialOption,
+		IamSelfServiceRequiresGrant: *iamopt.selfServiceRequiresGrant,
 	})
 	glog.V(0).Info("NewIamApiServer created")
 	if iamApiServer_err != nil {

weed/command/mini.go

@@ -556,6 +556,8 @@ func initMiniS3Flags() {
 	miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
 	miniS3Options.readerCacheSizeMB = cmdMini.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
 	miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
+	miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
 	// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
 	miniS3Options.debug = new(bool) // explicitly false
 	miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")

weed/command/s3.go

@@ -84,6 +84,8 @@ type S3Options struct {
 	readerCacheSizeMB         *int64
 
 	allowUntrustedRemoteEndpoints *bool
+	allowBackslashInKey           *bool
+	iamSelfServiceRequiresGrant   *bool
 	// shutdownCtx, when non-nil, tells startS3Server to gracefully shut down
 	// its HTTP/gRPC servers once the ctx is cancelled, in addition to on
 	// interrupt. Used by weed mini to orchestrate an ordered shutdown.
@@ -140,6 +142,8 @@ func init() {
 	s3StandaloneOptions.cacheSizeMB = cmdS3.Flag.Int64("cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
 	s3StandaloneOptions.readerCacheSizeMB = cmdS3.Flag.Int64("readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
 	s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage)
+	s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
 }
 
 var cmdS3 = &Command{
@@ -337,7 +341,8 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	router := mux.NewRouter().SkipClean(true)
 	var localFilerSocket string
-	if s3opt.localFilerSocket != nil {
+	// The filer only listens on its local socket outside Windows.
+	if s3opt.localFilerSocket != nil && runtime.GOOS != "windows" {
 		localFilerSocket = *s3opt.localFilerSocket
 	}
 	var s3ApiServer *s3api.S3ApiServer
@@ -399,6 +404,8 @@ func (s3opt *S3Options) startS3Server() bool {
 		MaxMB:                     filerMaxMB,
 
 		AllowUntrustedRemoteEndpoints: *s3opt.allowUntrustedRemoteEndpoints,
+		AllowBackslashInKey:           *s3opt.allowBackslashInKey,
+		IamSelfServiceRequiresGrant:   *s3opt.iamSelfServiceRequiresGrant,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)

weed/command/scaffold/filer.toml

@@ -33,12 +33,22 @@ recursive_delete = false
 # faster than previous leveldb, recommended.
 enabled = true
 dir = "./filerldb2"                    # directory to store level db files
+# optional tuning, applied to each of the 8 partitioned stores
+# block_cache_size_mb = 32             # sorted-table read cache
+# write_buffer_size_mb = 16            # memtable size before flushing to disk
+# block_size_kb = 4                    # uncompressed block ('page') size inside table files
+# compaction_table_size_mb = 2         # target size of generated table files
 
 [leveldb3]
 # similar to leveldb2.
 # each bucket has its own meta store.
 enabled = false
 dir = "./filerldb3"                    # directory to store level db files
+# optional tuning, applied to the default store and each bucket store
+# block_cache_size_mb = 32             # sorted-table read cache
+# write_buffer_size_mb = 16            # memtable size before flushing to disk
+# block_size_kb = 4                    # uncompressed block ('page') size inside table files
+# compaction_table_size_mb = 2         # target size of generated table files
 
 [rocksdb]
 # local on disk, similar to leveldb

weed/command/server.go

@@ -195,10 +195,12 @@ func init() {
 	s3Options.iamReadOnly = cmdServer.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
 	s3Options.cipher = cmdServer.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
 	s3Options.externalUrl = cmdServer.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Advertised to Iceberg and Lance clients, and tried first when verifying S3 signatures behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
+	s3Options.allowBackslashInKey = cmdServer.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
 	s3Options.defaultFileMode = cmdServer.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
 	s3Options.cacheSizeMB = cmdServer.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
 	s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
 	s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
+	s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
 
 	sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
 	sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
@@ -214,6 +216,7 @@ func init() {
 	sftpOptions.dataCenter = cmdServer.Flag.String("sftp.dataCenter", "", "prefer to read and write to volumes in this data center")
 	sftpOptions.localSocket = cmdServer.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
 	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
+	iamOptions.selfServiceRequiresGrant = s3Options.iamSelfServiceRequiresGrant
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
 	webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")

weed/command/shell.go

@@ -3,6 +3,7 @@ package command
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 
@@ -12,10 +13,11 @@ import (
 )
 
 var (
-	shellOptions      shell.ShellOptions
-	shellInitialFiler *string
-	shellCluster      *string
-	shellDebug        *bool
+	shellOptions        shell.ShellOptions
+	shellInitialFiler   *string
+	shellCluster        *string
+	shellDebug          *bool
+	shellConnectTimeout *time.Duration
 )
 
 func init() {
@@ -25,6 +27,7 @@ func init() {
 	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888")
 	shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
 	shellDebug = cmdShell.Flag.Bool("debug", false, "print informational logs to stderr")
+	shellConnectTimeout = cmdShell.Flag.Duration("connectTimeout", 30*time.Second, "timeout for the initial master connection and filer discovery")
 }
 
 var cmdShell = &Command{
@@ -64,6 +67,7 @@ func runShell(command *Command, args []string) bool {
 	}
 	shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
 	shellOptions.Debug = *shellDebug
+	shellOptions.ConnectTimeout = *shellConnectTimeout
 	if shellOptions.Debug {
 		fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
 	}

weed/command/upload.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -31,6 +32,7 @@ type UploadOptions struct {
 	diskType     *string
 	maxMB        *int
 	usePublicUrl *bool
+	concurrency  *int
 }
 
 func init() {
@@ -46,6 +48,7 @@ func init() {
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
 	upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
+	upload.concurrency = cmdUpload.Flag.Int("c", 8, "concurrent file upload goroutines when uploading a folder")
 }
 
 var cmdUpload = &Command{
@@ -85,6 +88,7 @@ func runUpload(cmd *Command, args []string) bool {
 		if *upload.dir == "" {
 			return false
 		}
+		var filePaths []string
 		err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
 			if err == nil {
 				if !info.IsDir() {
@@ -93,23 +97,7 @@ func runUpload(cmd *Command, args []string) bool {
 							return nil
 						}
 					}
-					parts, e := operation.NewFileParts([]string{path})
-					if e != nil {
-						return e
-					}
-					results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
-						Replication: *upload.replication,
-						Collection:  *upload.collection,
-						DataCenter:  *upload.dataCenter,
-						Ttl:         *upload.ttl,
-						DiskType:    *upload.diskType,
-						MaxMB:       *upload.maxMB,
-					}, *upload.usePublicUrl)
-					bytes, _ := json.Marshal(results)
-					fmt.Println(string(bytes))
-					if e != nil {
-						return e
-					}
+					filePaths = append(filePaths, path)
 				}
 			} else {
 				fmt.Println(err)
@@ -118,6 +106,72 @@ func runUpload(cmd *Command, args []string) bool {
 		})
 		if err != nil {
 			fmt.Println(err.Error())
+		}
+
+		concurrency := *upload.concurrency
+		if concurrency < 1 {
+			concurrency = 1
+		}
+		masterFn := func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }
+		pref := operation.StoragePreference{
+			Replication: *upload.replication,
+			Collection:  *upload.collection,
+			DataCenter:  *upload.dataCenter,
+			Ttl:         *upload.ttl,
+			DiskType:    *upload.diskType,
+			MaxMB:       *upload.maxMB,
+		}
+		results := make([][]operation.SubmitResult, len(filePaths))
+		ready := make([]chan struct{}, len(filePaths))
+		for index := range ready {
+			ready[index] = make(chan struct{})
+		}
+		printDone := make(chan bool, 1)
+		go func() {
+			failed := false
+			for index := range ready {
+				<-ready[index]
+				bytes, _ := json.Marshal(results[index])
+				fmt.Println(string(bytes))
+				for _, r := range results[index] {
+					if r.Error != "" {
+						failed = true
+					}
+				}
+			}
+			printDone <- failed
+		}()
+
+		sem := make(chan struct{}, concurrency)
+		var wg sync.WaitGroup
+		for index, filePath := range filePaths {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func(index int, filePath string) {
+				defer close(ready[index])
+				defer wg.Done()
+				defer func() { <-sem }()
+				parts, e := operation.NewFileParts([]string{filePath})
+				if e != nil {
+					results[index] = []operation.SubmitResult{{FileName: filepath.Base(filePath), Error: e.Error()}}
+					return
+				}
+				fileResults, e := operation.SubmitFiles(masterFn, grpcDialOption, parts, pref, *upload.usePublicUrl)
+				if e != nil {
+					for i := range fileResults {
+						if fileResults[i].Error == "" {
+							fileResults[i].Error = e.Error()
+						}
+					}
+				}
+				results[index] = fileResults
+			}(index, filePath)
+		}
+		wg.Wait()
+		if <-printDone {
+			return false
+		}
+		if err != nil {
 			return false
 		}
 	} else {
@@ -134,12 +188,12 @@ func runUpload(cmd *Command, args []string) bool {
 			DiskType:    *upload.diskType,
 			MaxMB:       *upload.maxMB,
 		}, *upload.usePublicUrl)
+		bytes, _ := json.Marshal(results)
+		fmt.Println(string(bytes))
 		if err != nil {
 			fmt.Println(err.Error())
 			return false
 		}
-		bytes, _ := json.Marshal(results)
-		fmt.Println(string(bytes))
 	}
 	return true
 }

weed/credential/credential_manager.go

@@ -15,6 +15,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 )
 
 // FilerAddressSetter is an interface for credential stores that need a dynamic filer address
@@ -168,15 +169,16 @@ func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_
 	staticNames := cm.staticNames
 	cm.staticMu.RUnlock()
 	if len(staticNames) > 0 {
-		var dynamicOnly []*iam_pb.Identity
-		for _, ident := range config.Identities {
+		configCopy := proto.Clone(config).(*iam_pb.S3ApiConfiguration)
+		dynamicOnly := configCopy.Identities[:0]
+		for _, ident := range configCopy.Identities {
 			if !staticNames[ident.Name] {
 				dynamicOnly = append(dynamicOnly, ident)
 			}
 		}
-		configCopy := *config
+		clear(configCopy.Identities[len(dynamicOnly):])
 		configCopy.Identities = dynamicOnly
-		return cm.Store.SaveConfiguration(ctx, &configCopy)
+		return cm.Store.SaveConfiguration(ctx, configCopy)
 	}
 	return cm.Store.SaveConfiguration(ctx, config)
 }

weed/credential/credential_manager_snapshot_test.go

@@ -0,0 +1,46 @@
+package credential
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+type snapshotConfigurationStore struct {
+	CredentialStore
+	saved *iam_pb.S3ApiConfiguration
+}
+
+func (s *snapshotConfigurationStore) SaveConfiguration(_ context.Context, config *iam_pb.S3ApiConfiguration) error {
+	s.saved = config
+	return nil
+}
+
+func TestSaveConfigurationFiltersStaticIdentitySnapshot(t *testing.T) {
+	t.Parallel()
+	store := &snapshotConfigurationStore{}
+	manager := &CredentialManager{Store: store}
+	manager.SetStaticIdentities([]*iam_pb.Identity{{Name: "static"}})
+	source := &iam_pb.S3ApiConfiguration{Identities: []*iam_pb.Identity{
+		{Name: "alice", Actions: []string{"Read"}},
+		{Name: "static"},
+		{Name: "bob", Actions: []string{"Write"}},
+	}}
+	before := proto.Clone(source)
+	if err := manager.SaveConfiguration(context.Background(), source); err != nil {
+		t.Fatal(err)
+	}
+	if store.saved == nil || len(store.saved.Identities) != 2 || store.saved.Identities[0].Name != "alice" || store.saved.Identities[1].Name != "bob" {
+		t.Fatalf("unexpected saved configuration: %v", store.saved)
+	}
+	if !proto.Equal(source, before) {
+		t.Fatal("filtering mutated the caller's configuration")
+	}
+	store.saved.Identities[0].Actions[0] = "Admin"
+	store.saved.Identities[1].Name = "changed"
+	if !proto.Equal(source, before) {
+		t.Fatal("saved identities alias the caller's configuration")
+	}
+}

weed/filer/abstract_sql/abstract_sql_store.go

@@ -23,14 +23,20 @@ type SqlGenerator interface {
 	GetSqlDeleteFolderChildren(tableName string) string
 	GetSqlListExclusive(tableName string) string
 	GetSqlListInclusive(tableName string) string
+	// GetSqlScanPage returns the query and arguments selecting dirhash, name,
+	// directory and meta of up to limit rows after the (dirhash, name) cursor.
+	GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{})
 	GetSqlCreateTable(tableName string) string
 	GetSqlDropTable(tableName string) string
 }
 
 type AbstractSqlStore struct {
 	SqlGenerator
-	DB                     *sql.DB
-	KvDB                   *sql.DB
+	DB   *sql.DB
+	KvDB *sql.DB
+	// ScanDB, when set, is the pool snapshot scans run on (SQLite's main pool
+	// is a single connection that a long scan must not hold).
+	ScanDB                 *sql.DB
 	SupportBucketTable     bool
 	SkipDDL                bool
 	dbs                    map[string]bool
@@ -457,6 +463,9 @@ func (store *AbstractSqlStore) Shutdown() {
 	if store.KvDB != nil {
 		store.KvDB.Close()
 	}
+	if store.ScanDB != nil {
+		store.ScanDB.Close()
+	}
 }
 
 func isValidBucket(bucket string) bool {

weed/filer/abstract_sql/abstract_sql_store_scan.go

@@ -0,0 +1,368 @@
+package abstract_sql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var _ filer.EntryScanner = (*AbstractSqlStore)(nil)
+
+// scanTable is one table to read during a scan together with the path its
+// directory column is relative to: "" for the default table, whose rows hold
+// full directories, and "/buckets/<bucket>" for a per-bucket table.
+type scanTable struct {
+	table string
+	base  string
+	// cachedOnly marks a table that only a stale bucket cache remembers: its
+	// bucket directory was already gone from the snapshot, so a missing table
+	// means it was dropped meanwhile, not that rows are absent.
+	cachedOnly bool
+}
+
+// scanPageSize is how many rows one keyset page of a scan fetches.
+var scanPageSize = 1000
+
+// SetScanPageSizeForTest lets tests drive the keyset cursor across pages. It
+// returns the previous size.
+func SetScanPageSizeForTest(n int) int {
+	prev := scanPageSize
+	scanPageSize = n
+	return prev
+}
+
+// ScanEntries implements filer.EntryScanner: every table that can hold an
+// entry below dirPath is read inside one REPEATABLE READ, read-only
+// transaction, in primary-key pages. All rows are seen as of one instant; a
+// rename committed meanwhile shows up under exactly one of its names. The
+// transaction takes no row locks and does not block writers; it does hold the
+// tables it has read against DDL, so a bucket deleted while the scan runs
+// drops its table once the scan ends. A page's cursor is closed before its
+// entries go to fn, so fn may read the store through the context it receives
+// (which carries the transaction) on the same connection.
+func (store *AbstractSqlStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
+	// Read before the transaction holds a connection: OnBucketCreation keeps
+	// dbsLock while it waits for a connection of its own, so taking the lock
+	// while holding a connection could deadlock on an exhausted pool.
+	cachedBuckets := store.cachedBucketNames()
+
+	tx, err := store.beginSnapshotTx(ctx)
+	if err != nil {
+		return err
+	}
+	// Read-only: nothing to commit, and a rollback releases the snapshot.
+	defer tx.Rollback()
+
+	tables, err := store.scanTables(ctx, tx, dirPath, cachedBuckets)
+	if err != nil {
+		return err
+	}
+	// Reads made through this context (getTxOrDB, kvDB) use the transaction
+	// and therefore the snapshot.
+	txCtx := context.WithValue(ctx, "tx", tx)
+	// The root's own row may live in a table the scan does not read (a
+	// bucket directory is in the default table, its contents in the bucket's).
+	if root, err := store.readRowInSnapshot(txCtx, tx, dirPath); err != nil {
+		return err
+	} else if root != nil {
+		if err := fn(txCtx, root); err != nil {
+			return err
+		}
+	}
+	for _, t := range tables {
+		if err := store.scanOneTable(txCtx, tx, t, dirPath, fn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// readRowInSnapshot reads one entry's row inside the transaction, routing to
+// the table getTxOrDB would use but without its bucket-table creation. A
+// missing row is nil.
+func (store *AbstractSqlStore) readRowInSnapshot(ctx context.Context, tx *sql.Tx, p util.FullPath) (*filer.Entry, error) {
+	if p == "/" {
+		return nil, nil
+	}
+	parent, name := p.DirAndName()
+	table, shortDir := DEFAULT_TABLE, parent
+	if store.SupportBucketTable {
+		if bucket := bucketTableFor(util.FullPath(parent)); bucket != "" {
+			table = bucket
+			if shortDir = strings.TrimPrefix(parent, bucketBase(bucket)); shortDir == "" {
+				shortDir = "/"
+			}
+		}
+	}
+	var data []byte
+	err := tx.QueryRowContext(ctx, store.GetSqlFind(table), util.HashStringToLong(shortDir), name, shortDir).Scan(&data)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, missingTableOr(err, table, fmt.Sprintf("read %s", p))
+	}
+	entry := &filer.Entry{FullPath: p}
+	if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
+		return nil, fmt.Errorf("decode %s: %v", p, err)
+	}
+	return entry, nil
+}
+
+// beginSnapshotTx starts the read-only REPEATABLE READ transaction that gives
+// one snapshot across statements. Nothing weaker is tried: under READ
+// COMMITTED each page would see a different snapshot, and the stream would
+// still be advertised as one.
+func (store *AbstractSqlStore) beginSnapshotTx(ctx context.Context) (*sql.Tx, error) {
+	db := store.DB
+	if store.ScanDB != nil {
+		db = store.ScanDB
+	} else if db.Stats().MaxOpenConnections == 1 {
+		// The scan would hold the filer's only connection for its whole run.
+		return nil, fmt.Errorf("%w: the store's single connection cannot be held for a scan", filer.ErrUnsupportedEntryScan)
+	}
+	// A BeginTx failure (a down database) must fail the traversal, not route it
+	// to the walk: a recovered database lets the walk read renamed-away entries
+	// as absent, the very loss this scan exists to prevent.
+	tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
+	if err != nil {
+		return nil, fmt.Errorf("begin snapshot: %v", err)
+	}
+	return tx, nil
+}
+
+func (store *AbstractSqlStore) cachedBucketNames() []string {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+	names := make([]string, 0, len(store.dbs))
+	for name := range store.dbs {
+		names = append(names, name)
+	}
+	return names
+}
+
+// scanTables picks the tables to read for dirPath: the default table alone,
+// one bucket's table for a path inside that bucket, or the default table plus
+// every bucket table (enumerated from /buckets inside the snapshot).
+func (store *AbstractSqlStore) scanTables(ctx context.Context, tx *sql.Tx, dirPath util.FullPath, cachedBuckets []string) ([]scanTable, error) {
+	if !store.SupportBucketTable {
+		return []scanTable{{table: DEFAULT_TABLE}}, nil
+	}
+	if bucket := bucketTableFor(dirPath); bucket != "" {
+		return []scanTable{{table: bucket, base: bucketBase(bucket)}}, nil
+	}
+	tables := []scanTable{{table: DEFAULT_TABLE}}
+	if dirPath != "/" && string(dirPath) != bucketsRoot {
+		// Outside every bucket (e.g. /topics, or an internal /buckets/.system
+		// folder): everything below dirPath is in the default table.
+		return tables, nil
+	}
+	buckets, err := store.listBucketNames(ctx, tx, cachedBuckets)
+	if err != nil {
+		return nil, err
+	}
+	for _, bucket := range buckets.names {
+		tables = append(tables, scanTable{table: bucket, base: bucketBase(bucket), cachedOnly: !buckets.seen[bucket]})
+	}
+	return tables, nil
+}
+
+const bucketsRoot = "/buckets"
+
+func bucketBase(bucket string) string {
+	return bucketsRoot + "/" + bucket
+}
+
+// bucketTableFor returns the bucket whose table alone holds everything below
+// dirPath, or "" when dirPath is not inside a real bucket (as getTxOrDB routes).
+func bucketTableFor(dirPath util.FullPath) string {
+	p := string(dirPath)
+	if !strings.HasPrefix(p, bucketsRoot+"/") {
+		return ""
+	}
+	rest := p[len(bucketsRoot)+1:]
+	bucket := rest
+	if i := strings.Index(rest, "/"); i >= 0 {
+		bucket = rest[:i]
+	}
+	if bucket == "" || strings.HasPrefix(bucket, ".") || !isValidBucket(bucket) {
+		return ""
+	}
+	return bucket
+}
+
+// bucketNames is the snapshot-listed and cached-only buckets whose tables a
+// scan may read.
+type bucketNames struct {
+	names []string
+	seen  map[string]bool // listed under /buckets inside the snapshot
+}
+
+// listBucketNames reads the directories under /buckets within the snapshot
+// and keeps those that route to their own table, plus the buckets this
+// process has already seen (their directory entry may be gone, their rows not).
+func (store *AbstractSqlStore) listBucketNames(ctx context.Context, tx *sql.Tx, cachedBuckets []string) (bucketNames, error) {
+	var names bucketNames
+	names.seen = make(map[string]bool)
+	added := make(map[string]bool)
+	add := func(name string) {
+		if added[name] || strings.HasPrefix(name, ".") || !isValidBucket(name) {
+			return
+		}
+		added[name] = true
+		names.names = append(names.names, name)
+	}
+
+	const page = 1000
+	startFrom := ""
+	for {
+		type row struct {
+			name string
+			data []byte
+		}
+		var rows []row
+		if err := func() error {
+			res, err := tx.QueryContext(ctx, store.GetSqlListExclusive(DEFAULT_TABLE), util.HashStringToLong(bucketsRoot), startFrom, bucketsRoot, "%", page)
+			if err != nil {
+				return fmt.Errorf("list %s: %v", bucketsRoot, err)
+			}
+			defer res.Close()
+			for res.Next() {
+				var r row
+				if err := res.Scan(&r.name, &r.data); err != nil {
+					return fmt.Errorf("scan %s: %v", bucketsRoot, err)
+				}
+				rows = append(rows, r)
+			}
+			return res.Err()
+		}(); err != nil {
+			return bucketNames{}, err
+		}
+		for _, r := range rows {
+			startFrom = r.name
+			entry := &filer.Entry{FullPath: util.NewFullPath(bucketsRoot, r.name)}
+			if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
+				return bucketNames{}, fmt.Errorf("scan decode %s: %v", entry.FullPath, err)
+			}
+			if entry.IsDirectory() {
+				names.seen[r.name] = true
+				add(r.name)
+			}
+		}
+		if len(rows) < page {
+			break
+		}
+	}
+
+	for _, name := range cachedBuckets {
+		add(name)
+	}
+	return names, nil
+}
+
+// scanOneTable reads one table in primary-key pages and hands the entries
+// strictly below dirPath to fn (the root itself is read separately). Key-value
+// records share the default table; GenDirAndName stores the raw key bytes as
+// their dirhash while entry rows hash the directory, so the mismatch
+// identifies them exactly.
+func (store *AbstractSqlStore) scanOneTable(ctx context.Context, tx *sql.Tx, t scanTable, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
+	type row struct {
+		dirHash   int64
+		name, dir string
+		data      []byte
+	}
+	cursorHash, cursorName := int64(math.MinInt64), ""
+	firstPage := true
+	for {
+		var page []row
+		if t.cachedOnly && firstPage {
+			// A missing table can only be met by the first page read: once a
+			// row came back the tx holds the table against a drop. On
+			// PostgreSQL the failed query aborts the transaction, so the skip
+			// needs a savepoint to roll back to before the next table's read.
+			if _, err := tx.ExecContext(ctx, "SAVEPOINT seaweed_scan_bucket"); err != nil {
+				return fmt.Errorf("scan table %s: %v", t.table, err)
+			}
+		}
+		sqlText, args := store.GetSqlScanPage(t.table, cursorHash, cursorName, scanPageSize)
+		rows, err := tx.QueryContext(ctx, sqlText, args...)
+		if err != nil {
+			if t.cachedOnly && isMissingTableError(err) {
+				tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT seaweed_scan_bucket")
+				return nil // a cached-only name whose table was dropped meanwhile
+			}
+			return missingTableOr(err, t.table, fmt.Sprintf("scan table %s", t.table))
+		}
+		firstPage = false
+		if err := func() error {
+			defer rows.Close()
+			for rows.Next() {
+				var r row
+				if err := rows.Scan(&r.dirHash, &r.name, &r.dir, &r.data); err != nil {
+					return fmt.Errorf("scan table %s: %v", t.table, err)
+				}
+				page = append(page, r)
+			}
+			return rows.Err()
+		}(); err != nil {
+			return err
+		}
+
+		for _, r := range page {
+			cursorHash, cursorName = r.dirHash, r.name
+			if r.dirHash != util.HashStringToLong(r.dir) {
+				continue // a key-value record, see GenDirAndName
+			}
+			fullPath := util.NewFullPath(t.base+r.dir, r.name)
+			if !isStrictlyBelow(fullPath, dirPath) {
+				continue
+			}
+			entry := &filer.Entry{FullPath: fullPath}
+			if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
+				return fmt.Errorf("scan decode %s: %v", fullPath, err)
+			}
+			if err := fn(ctx, entry); err != nil {
+				return err
+			}
+		}
+		if len(page) < scanPageSize {
+			return nil
+		}
+	}
+}
+
+// missingTableOr names the condition when a bucket's table is not there. The
+// scan fails closed: from inside the snapshot a table that was never created
+// cannot be told from one dropped a moment ago, whose rows the snapshot still
+// counts, and treating either as empty would let fsck purge referenced chunks.
+func missingTableOr(err error, table, what string) error {
+	if table != DEFAULT_TABLE && isMissingTableError(err) {
+		return fmt.Errorf("bucket %s has a directory entry but no table (never created, or dropped during the scan): %v", table, err)
+	}
+	return fmt.Errorf("%s: %v", what, err)
+}
+
+// isMissingTableError recognizes a missing table by message, as InsertEntry
+// recognizes duplicates: PostgreSQL `relation "x" does not exist` (SQLSTATE
+// 42P01), MySQL error 1146 (SQLSTATE 42S02), SQLite "no such table".
+func isMissingTableError(err error) bool {
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "42p01") ||
+		(strings.Contains(msg, "relation \"") && strings.Contains(msg, "does not exist")) ||
+		strings.Contains(msg, "42s02") ||
+		strings.Contains(msg, "error 1146") ||
+		strings.Contains(msg, "no such table")
+}
+
+// isStrictlyBelow reports whether p is a descendant of dir (not dir itself).
+func isStrictlyBelow(p, dir util.FullPath) bool {
+	if dir == "/" {
+		return p != "/"
+	}
+	return strings.HasPrefix(string(p), string(dir)+"/")
+}

weed/filer/abstract_sql/abstract_sql_store_scan_test.go

@@ -0,0 +1,51 @@
+package abstract_sql
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestBucketTableFor(t *testing.T) {
+	cases := map[string]string{
+		"/":                         "",
+		"/buckets":                  "",
+		"/buckets/":                 "",
+		"/buckets/mybucket":         "mybucket",
+		"/buckets/mybucket/a/b":     "mybucket",
+		"/buckets/.system/owners":   "",
+		"/buckets/Invalid_Bucket/x": "",
+		"/topics/.system/log":       "",
+		"/bucketsfoo/mybucket":      "",
+	}
+	for path, want := range cases {
+		if got := bucketTableFor(util.FullPath(path)); got != want {
+			t.Errorf("bucketTableFor(%s) = %q, want %q", path, got, want)
+		}
+	}
+}
+
+// Only a missing table may be skipped; any other schema error must fail the
+// scan, or a bucket's live chunks would be reported as orphans.
+func TestIsMissingTableError(t *testing.T) {
+	for _, msg := range []string{
+		`ERROR: relation "ghost" does not exist (SQLSTATE 42P01)`,
+		`Error 1146 (42S02): Table 'seaweedfs.ghost' doesn't exist`,
+		`SQL logic error: no such table: ghost (1)`,
+	} {
+		if !isMissingTableError(errors.New(msg)) {
+			t.Errorf("not recognized as a missing table: %s", msg)
+		}
+	}
+	for _, msg := range []string{
+		`ERROR: column "directory" does not exist (SQLSTATE 42703)`,
+		`Error 1054 (42S22): Unknown column 'directory' in 'field list'`,
+		`SQL logic error: no such column: directory (1)`,
+		"connection refused",
+	} {
+		if isMissingTableError(errors.New(msg)) {
+			t.Errorf("wrongly taken for a missing table: %s", msg)
+		}
+	}
+}

weed/filer/filer.go

@@ -416,6 +416,28 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
 	return f.Store.RollbackTransaction(ctx)
 }
 
+type syncWriteKey struct{}
+
+// WithSyncWrite marks the entry writes under ctx durable-on-ack: a store that
+// buffers writes by default must flush them before returning.
+func WithSyncWrite(ctx context.Context) context.Context {
+	return context.WithValue(ctx, syncWriteKey{}, true)
+}
+
+// SyncWrite reports whether ctx carries a durability request for entry writes.
+func SyncWrite(ctx context.Context) bool {
+	return ctx.Value(syncWriteKey{}) != nil
+}
+
+// syncWriteContext flags ctx when the storage rule on p asks for fsync, so the
+// store write is held to the same durability as the data path.
+func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context {
+	if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync {
+		return WithSyncWrite(ctx)
+	}
+	return ctx
+}
+
 // CreateEntry creates or replaces an entry. When existing is non-nil the caller
 // has already fetched the current entry at this path under a path lock, and it
 // is reused instead of looking the store up again; pass nil to have CreateEntry
@@ -426,6 +448,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
 		return nil
 	}
 
+	ctx = f.syncWriteContext(ctx, entry.FullPath)
+
 	if entry.FullPath.IsLongerFileName(maxFilenameLength) {
 		return filer_pb.ErrEntryNameTooLong
 	}
@@ -631,6 +655,7 @@ func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath)
 // deleted with, so a restore cannot hand back a directory more permissive than the one
 // it replaces - including when someone else already put it back with wider ones.
 func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error {
+	ctx = f.syncWriteContext(ctx, dirPath)
 	existing, err := f.FindEntry(ctx, dirPath)
 	if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return err
@@ -680,6 +705,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
 }
 
 func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
+	ctx = f.syncWriteContext(ctx, entry.FullPath)
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.Attr.Inode != 0 {

weed/filer/filer_delete_entry.go

@@ -122,6 +122,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return nil
 	}
 
+	ctx = f.syncWriteContext(ctx, p)
+
 	entry, findErr := f.FindEntry(ctx, p)
 	if findErr != nil {
 		return findErr

weed/filer/filerstore.go

@@ -44,6 +44,22 @@ type FilerStore interface {
 	Shutdown()
 }
 
+// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before
+// any entry is visited, when the store cannot serve the scan.
+var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store")
+
+// EntryScanner is an optional FilerStore capability: visit every entry at and
+// below dirPath, in any order, from one consistent snapshot of the store.
+//
+// A directory walk sees each directory at a different time, so an entry
+// renamed from a directory not yet listed into one already listed is never
+// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it
+// exactly once. The context handed to fn carries the snapshot, so store reads
+// made with it (hard-link records) see the same instant.
+type EntryScanner interface {
+	ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error
+}
+
 type BucketAware interface {
 	OnBucketCreation(bucket string)
 	OnBucketDeletion(bucket string)

weed/filer/filerstore_wrapper.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -19,6 +20,7 @@ import (
 var (
 	_ = VirtualFilerStore(&FilerStoreWrapper{})
 	_ = Debuggable(&FilerStoreWrapper{})
+	_ = EntryScanner(&FilerStoreWrapper{})
 )
 
 type VirtualFilerStore interface {
@@ -402,6 +404,33 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 	return lastFileName, err
 }
 
+// ScanEntries delegates to the default store. With path-specific stores
+// configured, entries below dirPath could live in several stores and one
+// snapshot per store is not one snapshot, so the scan is declined. Hard links
+// are resolved inside the snapshot; a store error fails the scan (the per-path
+// copy can be stale), a missing record falls back to it as listings do.
+func (fsw *FilerStoreWrapper) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error {
+	if fsw.hasPathSpecificStore {
+		return ErrUnsupportedEntryScan
+	}
+	scanner, ok := fsw.defaultStore.(EntryScanner)
+	if !ok {
+		return ErrUnsupportedEntryScan
+	}
+	stats.FilerStoreCounter.WithLabelValues(fsw.defaultStore.GetName(), "scan").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.defaultStore.GetName(), "scan").Observe(time.Since(start).Seconds())
+	}()
+	return scanner.ScanEntries(ctx, dirPath, func(scanCtx context.Context, entry *Entry) error {
+		if err := fsw.maybeReadHardLink(scanCtx, entry); err != nil && !errors.Is(err, ErrKvNotFound) {
+			return fmt.Errorf("resolve hard link of %s: %w", entry.FullPath, err)
+		}
+		filer_pb.AfterEntryDeserialization(entry.GetChunks())
+		return fn(scanCtx, entry)
+	})
+}
+
 func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
 	actualStore := fsw.getActualStore(dirPath + "/")
 

weed/filer/leveldb/leveldb_store.go

@@ -79,6 +79,15 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
 	return nil
 }
 
+var syncWriteOptions = &opt.WriteOptions{Sync: true}
+
+func writeOptions(ctx context.Context) *opt.WriteOptions {
+	if filer.SyncWrite(ctx) {
+		return syncWriteOptions
+	}
+	return nil
+}
+
 func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
 	key := genKey(entry.DirAndName())
 
@@ -91,7 +100,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry)
 		value = weed_util.MaybeGzipData(value)
 	}
 
-	err = store.db.Put(key, value, nil)
+	err = store.db.Put(key, value, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -168,7 +177,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful
 func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	key := genKey(fullpath.DirAndName())
 
-	err = store.db.Delete(key, nil)
+	err = store.db.Delete(key, writeOptions(ctx))
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
@@ -195,7 +204,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
 	}
 	iter.Release()
 
-	err = store.db.Write(batch, nil)
+	err = store.db.Write(batch, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)

weed/filer/leveldb2/leveldb2_store.go

@@ -6,6 +6,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"math"
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -28,6 +29,11 @@ type LevelDB2Store struct {
 	dbs      []*leveldb.DB
 	dbCount  int
 	ReadOnly bool
+
+	blockCacheSizeMB      int
+	writeBufferSizeMB     int
+	blockSizeKB           int
+	compactionTableSizeMB int
 }
 
 func (store *LevelDB2Store) GetName() string {
@@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string {
 
 func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
 	dir := configuration.GetString(prefix + "dir")
+	store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
+	store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
+	store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
+	store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
+	if err := store.validateSizeOptions(); err != nil {
+		return err
+	}
 	return store.initialize(dir, 8)
 }
 
+// validateSizeOptions rejects sizes too large to convert to bytes without
+// overflowing int on this platform.
+func (store *LevelDB2Store) validateSizeOptions() error {
+	for name, kv := range map[string][2]int{
+		"block_cache_size_mb":      {store.blockCacheSizeMB, 1 << 20},
+		"write_buffer_size_mb":     {store.writeBufferSizeMB, 1 << 20},
+		"block_size_kb":            {store.blockSizeKB, 1 << 10},
+		"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
+	} {
+		if kv[0] > math.MaxInt/kv[1] {
+			return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0])
+		}
+	}
+	return nil
+}
+
+func (store *LevelDB2Store) buildOptions() *opt.Options {
+	opts := &opt.Options{
+		BlockCacheCapacity: 32 * 1024 * 1024,         // default value is 8MiB
+		WriteBuffer:        16 * 1024 * 1024,         // default value is 4MiB
+		Filter:             filter.NewBloomFilter(8), // false positive rate 0.02
+		ReadOnly:           store.ReadOnly,
+	}
+	if store.blockCacheSizeMB > 0 {
+		opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
+	}
+	if store.writeBufferSizeMB > 0 {
+		opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
+	}
+	if store.blockSizeKB > 0 {
+		opts.BlockSize = store.blockSizeKB * 1024
+	}
+	if store.compactionTableSizeMB > 0 {
+		opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
+	}
+	return opts
+}
+
 func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 	glog.Infof("filer store leveldb2 dir: %s", dir)
 	os.MkdirAll(dir, 0755)
@@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
-	opts := &opt.Options{
-		BlockCacheCapacity: 32 * 1024 * 1024,         // default value is 8MiB
-		WriteBuffer:        16 * 1024 * 1024,         // default value is 4MiB
-		Filter:             filter.NewBloomFilter(8), // false positive rate 0.02
-		ReadOnly:           store.ReadOnly,
-	}
+	opts := store.buildOptions()
 
 	for d := 0; d < dbCount; d++ {
 		dbFolder := fmt.Sprintf("%s/%02d", dir, d)
@@ -81,6 +127,15 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
 	return nil
 }
 
+var syncWriteOptions = &opt.WriteOptions{Sync: true}
+
+func writeOptions(ctx context.Context) *opt.WriteOptions {
+	if filer.SyncWrite(ctx) {
+		return syncWriteOptions
+	}
+	return nil
+}
+
 func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
 	dir, name := entry.DirAndName()
 	key, partitionId := genKey(dir, name, store.dbCount)
@@ -94,7 +149,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry)
 		value = weed_util.MaybeGzipData(value)
 	}
 
-	err = store.dbs[partitionId].Put(key, value, nil)
+	err = store.dbs[partitionId].Put(key, value, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -140,7 +195,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	dir, name := fullpath.DirAndName()
 	key, partitionId := genKey(dir, name, store.dbCount)
 
-	err = store.dbs[partitionId].Delete(key, nil)
+	err = store.dbs[partitionId].Delete(key, writeOptions(ctx))
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
@@ -167,7 +222,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
 	}
 	iter.Release()
 
-	err = store.dbs[partitionId].Write(batch, nil)
+	err = store.dbs[partitionId].Write(batch, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)

weed/filer/leveldb2/leveldb2_store_test.go

@@ -2,6 +2,8 @@ package leveldb
 
 import (
 	"context"
+	"math"
+	"strings"
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) {
 
 }
 
+type fakeConfiguration map[string]interface{}
+
+func (c fakeConfiguration) GetString(key string) string {
+	value, _ := c[key].(string)
+	return value
+}
+
+func (c fakeConfiguration) GetBool(key string) bool {
+	value, _ := c[key].(bool)
+	return value
+}
+
+func (c fakeConfiguration) GetInt(key string) int {
+	value, _ := c[key].(int)
+	return value
+}
+
+func (c fakeConfiguration) GetStringSlice(key string) []string {
+	value, _ := c[key].([]string)
+	return value
+}
+
+func (c fakeConfiguration) SetDefault(key string, value interface{}) {
+	if _, found := c[key]; !found {
+		c[key] = value
+	}
+}
+
+func TestInitializeOptions(t *testing.T) {
+	store := &LevelDB2Store{}
+	if err := store.Initialize(fakeConfiguration{
+		"leveldb2.dir":                      t.TempDir(),
+		"leveldb2.block_cache_size_mb":      64,
+		"leveldb2.write_buffer_size_mb":     32,
+		"leveldb2.block_size_kb":            16,
+		"leveldb2.compaction_table_size_mb": 8,
+	}, "leveldb2."); err != nil {
+		t.Fatalf("initialize: %v", err)
+	}
+	defer store.Shutdown()
+
+	opts := store.buildOptions()
+	if opts.BlockCacheCapacity != 64*1024*1024 {
+		t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity)
+	}
+	if opts.WriteBuffer != 32*1024*1024 {
+		t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer)
+	}
+	if opts.BlockSize != 16*1024 {
+		t.Errorf("block size %d, want 16KiB", opts.BlockSize)
+	}
+	if opts.CompactionTableSize != 8*1024*1024 {
+		t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize)
+	}
+}
+
+func TestInitializeDefaultOptions(t *testing.T) {
+	store := &LevelDB2Store{}
+	if err := store.Initialize(fakeConfiguration{
+		"leveldb2.dir":                  t.TempDir(),
+		"leveldb2.block_cache_size_mb":  -1,
+		"leveldb2.write_buffer_size_mb": 0,
+	}, "leveldb2."); err != nil {
+		t.Fatalf("initialize: %v", err)
+	}
+	defer store.Shutdown()
+
+	opts := store.buildOptions()
+	if opts.BlockCacheCapacity != 32*1024*1024 {
+		t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
+	}
+	if opts.WriteBuffer != 16*1024*1024 {
+		t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
+	}
+}
+
+func TestInitializeRejectsOversizedOption(t *testing.T) {
+	store := &LevelDB2Store{}
+	err := store.Initialize(fakeConfiguration{
+		"leveldb2.dir":                  t.TempDir(),
+		"leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
+	}, "leveldb2.")
+	if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
+		t.Fatalf("expected overflow rejection, got %v", err)
+	}
+}
+
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
 	dir := t.TempDir()

weed/filer/leveldb3/leveldb3_store.go

@@ -6,6 +6,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"strings"
 	"sync"
@@ -35,6 +36,11 @@ type LevelDB3Store struct {
 	dbs      map[string]*leveldb.DB
 	dbsLock  sync.RWMutex
 	ReadOnly bool
+
+	blockCacheSizeMB      int
+	writeBufferSizeMB     int
+	blockSizeKB           int
+	compactionTableSizeMB int
 }
 
 func (store *LevelDB3Store) GetName() string {
@@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string {
 
 func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
 	dir := configuration.GetString(prefix + "dir")
+	store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
+	store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
+	store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
+	store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
+	if err := store.validateSizeOptions(); err != nil {
+		return err
+	}
 	return store.initialize(dir)
 }
 
+// validateSizeOptions rejects sizes too large to convert to bytes without
+// overflowing int on this platform.
+func (store *LevelDB3Store) validateSizeOptions() error {
+	for name, kv := range map[string][2]int{
+		"block_cache_size_mb":      {store.blockCacheSizeMB, 1 << 20},
+		"write_buffer_size_mb":     {store.writeBufferSizeMB, 1 << 20},
+		"block_size_kb":            {store.blockSizeKB, 1 << 10},
+		"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
+	} {
+		if kv[0] > math.MaxInt/kv[1] {
+			return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0])
+		}
+	}
+	return nil
+}
+
+func (store *LevelDB3Store) buildOptions(name string) *opt.Options {
+	opts := &opt.Options{
+		BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:        16 * 1024 * 1024, // default value is 4MiB
+		Filter:             filter.NewBloomFilter(8),
+		ReadOnly:           store.ReadOnly,
+	}
+	if name != DEFAULT {
+		opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB
+		opts.WriteBuffer = 8 * 1024 * 1024         // default value is 4MiB
+	}
+	if store.blockCacheSizeMB > 0 {
+		opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
+	}
+	if store.writeBufferSizeMB > 0 {
+		opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
+	}
+	if store.blockSizeKB > 0 {
+		opts.BlockSize = store.blockSizeKB * 1024
+	}
+	if store.compactionTableSizeMB > 0 {
+		opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
+	}
+	return opts
+}
+
 func (store *LevelDB3Store) initialize(dir string) (err error) {
 	glog.Infof("filer store leveldb3 dir: %s", dir)
 	os.MkdirAll(dir, 0755)
@@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
 }
 
 func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
-	bloom := filter.NewBloomFilter(8) // false positive rate 0.02
-	opts := &opt.Options{
-		BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
-		WriteBuffer:        16 * 1024 * 1024, // default value is 4MiB
-		Filter:             bloom,
-		ReadOnly:           store.ReadOnly,
-	}
-	if name != DEFAULT {
-		opts = &opt.Options{
-			BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
-			WriteBuffer:        8 * 1024 * 1024,  // default value is 4MiB
-			Filter:             bloom,
-			ReadOnly:           store.ReadOnly,
-		}
-	}
+	opts := store.buildOptions(name)
 
 	dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
 	os.MkdirAll(dbFolder, 0755)
@@ -180,6 +221,15 @@ func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
 	return nil
 }
 
+var syncWriteOptions = &opt.WriteOptions{Sync: true}
+
+func writeOptions(ctx context.Context) *opt.WriteOptions {
+	if filer.SyncWrite(ctx) {
+		return syncWriteOptions
+	}
+	return nil
+}
+
 func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
 
 	db, _, shortPath, err := store.findDB(entry.FullPath, false)
@@ -199,7 +249,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry)
 		value = weed_util.MaybeGzipData(value)
 	}
 
-	err = db.Put(key, value, nil)
+	err = db.Put(key, value, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -257,7 +307,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	dir, name := shortPath.DirAndName()
 	key := genKey(dir, name)
 
-	err = db.Delete(key, nil)
+	err = db.Delete(key, writeOptions(ctx))
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
@@ -298,7 +348,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w
 	}
 	iter.Release()
 
-	err = db.Write(batch, nil)
+	err = db.Write(batch, writeOptions(ctx))
 
 	if err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)

weed/filer/leveldb3/leveldb3_store_test.go

@@ -2,6 +2,8 @@ package leveldb
 
 import (
 	"context"
+	"math"
+	"strings"
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) {
 
 }
 
+type fakeConfiguration map[string]interface{}
+
+func (c fakeConfiguration) GetString(key string) string {
+	value, _ := c[key].(string)
+	return value
+}
+
+func (c fakeConfiguration) GetBool(key string) bool {
+	value, _ := c[key].(bool)
+	return value
+}
+
+func (c fakeConfiguration) GetInt(key string) int {
+	value, _ := c[key].(int)
+	return value
+}
+
+func (c fakeConfiguration) GetStringSlice(key string) []string {
+	value, _ := c[key].([]string)
+	return value
+}
+
+func (c fakeConfiguration) SetDefault(key string, value interface{}) {
+	if _, found := c[key]; !found {
+		c[key] = value
+	}
+}
+
+func TestInitializeOptions(t *testing.T) {
+	store := &LevelDB3Store{}
+	if err := store.Initialize(fakeConfiguration{
+		"leveldb3.dir":                      t.TempDir(),
+		"leveldb3.block_cache_size_mb":      64,
+		"leveldb3.write_buffer_size_mb":     32,
+		"leveldb3.block_size_kb":            16,
+		"leveldb3.compaction_table_size_mb": 8,
+	}, "leveldb3."); err != nil {
+		t.Fatalf("initialize: %v", err)
+	}
+	defer store.Shutdown()
+
+	for _, name := range []string{DEFAULT, "bucket-a"} {
+		opts := store.buildOptions(name)
+		if opts.BlockCacheCapacity != 64*1024*1024 {
+			t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity)
+		}
+		if opts.WriteBuffer != 32*1024*1024 {
+			t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer)
+		}
+		if opts.BlockSize != 16*1024 {
+			t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize)
+		}
+		if opts.CompactionTableSize != 8*1024*1024 {
+			t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize)
+		}
+	}
+}
+
+func TestInitializeDefaultOptions(t *testing.T) {
+	store := &LevelDB3Store{}
+	if err := store.Initialize(fakeConfiguration{
+		"leveldb3.dir":                  t.TempDir(),
+		"leveldb3.block_cache_size_mb":  -1,
+		"leveldb3.write_buffer_size_mb": 0,
+	}, "leveldb3."); err != nil {
+		t.Fatalf("initialize: %v", err)
+	}
+	defer store.Shutdown()
+
+	opts := store.buildOptions(DEFAULT)
+	if opts.BlockCacheCapacity != 32*1024*1024 {
+		t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
+	}
+	if opts.WriteBuffer != 16*1024*1024 {
+		t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
+	}
+
+	bucketOpts := store.buildOptions("bucket-a")
+	if bucketOpts.BlockCacheCapacity != 16*1024*1024 {
+		t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity)
+	}
+	if bucketOpts.WriteBuffer != 8*1024*1024 {
+		t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer)
+	}
+}
+
+func TestInitializeRejectsOversizedOption(t *testing.T) {
+	store := &LevelDB3Store{}
+	err := store.Initialize(fakeConfiguration{
+		"leveldb3.dir":                  t.TempDir(),
+		"leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
+	}, "leveldb3.")
+	if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
+		t.Fatalf("expected overflow rejection, got %v", err)
+	}
+}
+
 func TestEmptyRoot(t *testing.T) {
 	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
 	dir := t.TempDir()

weed/filer/mysql/mysql_sql_gen.go

@@ -62,6 +62,15 @@ func (gen *SqlGenMysql) nameExpr() string {
 	return "`name`"
 }
 
+// GetSqlScanPage spells the cursor as two primary-key ranges: MySQL 5.7 does
+// not turn a row-value comparison into a range seek. The name keeps the
+// column's own collation (BINARY would filesort every page); the scan promises
+// no lexical order. SQLite overrides this method.
+func (gen *SqlGenMysql) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
+	return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE `dirhash` > ? OR (`dirhash` = ? AND `name` > ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
+		[]interface{}{dirhash, dirhash, name, limit}
+}
+
 func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
 	name := gen.nameExpr()
 	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)

weed/filer/postgres/postgres_sql_gen.go

@@ -75,6 +75,14 @@ func (gen *SqlGenPostgres) nameExpr() string {
 	return "name"
 }
 
+// GetSqlScanPage uses a row-value comparison, one primary-key range seek on
+// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would
+// sort every page); the scan promises no lexical order.
+func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
+	return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName),
+		[]interface{}{dirhash, name, limit}
+}
+
 func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
 	name := gen.nameExpr()
 	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)

weed/filer/sqlite/sqlite_store.go

@@ -30,6 +30,31 @@ func (store *SqliteStore) GetName() string {
 	return "sqlite"
 }
 
+// sqliteSqlGen is the MySQL generator with SQLite's own page cursor: SQLite
+// seeks a row-value comparison on the primary key but scans the OR form.
+type sqliteSqlGen struct {
+	*mysql.SqlGenMysql
+}
+
+func (gen sqliteSqlGen) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
+	return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE (`dirhash`, `name`) > (?, ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
+		[]interface{}{dirhash, name, limit}
+}
+
+// ScanEntries offers the snapshot scan only in WAL mode: with a rollback
+// journal a long read transaction holds a shared lock and writers fail after
+// the busy timeout.
+func (store *SqliteStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
+	var mode string
+	if err := store.DB.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil {
+		return fmt.Errorf("%w: journal_mode: %v", filer.ErrUnsupportedEntryScan, err)
+	}
+	if !strings.EqualFold(mode, "wal") {
+		return fmt.Errorf("%w: sqlite journal_mode is %q, the scan needs wal", filer.ErrUnsupportedEntryScan, mode)
+	}
+	return store.AbstractSqlStore.ScanEntries(ctx, dirPath, fn)
+}
+
 func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	dbFile := configuration.GetString(prefix + "dbFile")
 	createTable := `CREATE TABLE IF NOT EXISTS "%s" (
@@ -69,11 +94,11 @@ func sqliteDSN(dbFile string) string {
 func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) {
 
 	store.SupportBucketTable = true
-	store.SqlGenerator = &mysql.SqlGenMysql{
+	store.SqlGenerator = sqliteSqlGen{&mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   "drop table if exists `%s`",
 		UpsertQueryTemplate:    upsertQuery,
-	}
+	}}
 
 	dsn := sqliteDSN(dbFile)
 
@@ -94,6 +119,11 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
 	}, 1, 1, 0); err != nil {
 		return err
 	}
+	// The main pool is one connection, which a long scan must not hold.
+	if store.ScanDB, err = sql.Open("sqlite", dsn); err != nil {
+		return fmt.Errorf("open scan connection to %s: %v", dbFile, err)
+	}
+	store.ScanDB.SetMaxOpenConns(1)
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
 		return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)

weed/filer/sqlite/sqlite_store_scan_test.go

@@ -0,0 +1,288 @@
+//go:build (linux || darwin || windows) && sqlite
+
+package sqlite
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// A rename committed while a scan is running is seen under exactly one of its
+// names. The scan reads from one snapshot, so unlike a directory walk it can
+// neither miss the entry when it moves into a directory already listed, nor
+// count it twice when it moves into one not listed yet.
+func TestScanEntriesSeesConcurrentRenameExactlyOnce(t *testing.T) {
+	dbFile := filepath.Join(t.TempDir(), "filer.db")
+	reader := &SqliteStore{}
+	if err := reader.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer reader.Shutdown()
+	// WAL lets the writer commit while the scan's read transaction is open.
+	if _, err := reader.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		t.Fatal(err)
+	}
+	writer := &SqliteStore{}
+	if err := writer.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer writer.Shutdown()
+
+	ctx := context.Background()
+	mk := func(p string) *filer.Entry {
+		return &filer.Entry{FullPath: util.FullPath(p), Attr: filer.Attr{Mode: 0644, Mtime: time.Now(), Crtime: time.Now()}}
+	}
+	for _, p := range []string{"/ckpt/a", "/ckpt/b", "/ckpt/tmp/x"} {
+		if err := reader.InsertEntry(ctx, mk(p)); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	defer func(n int) { abstract_sql.SetScanPageSizeForTest(n) }(abstract_sql.SetScanPageSizeForTest(2))
+
+	seen := map[string]int{}
+	renamed := false
+	err := reader.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
+		seen[string(e.FullPath)]++
+		if renamed {
+			return nil
+		}
+		renamed = true
+		// /ckpt/tmp/x -> /ckpt/x, committed by another connection mid-scan.
+		if err := writer.InsertEntry(ctx, mk("/ckpt/x")); err != nil {
+			return err
+		}
+		return writer.DeleteEntry(ctx, util.FullPath("/ckpt/tmp/x"))
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !renamed {
+		t.Fatal("scan yielded nothing")
+	}
+	if seen["/ckpt/a"] != 1 || seen["/ckpt/b"] != 1 {
+		t.Errorf("untouched entries seen %d and %d times, want once each", seen["/ckpt/a"], seen["/ckpt/b"])
+	}
+	if old, new := seen["/ckpt/tmp/x"], seen["/ckpt/x"]; old+new != 1 {
+		t.Errorf("renamed entry seen %d time(s) under the old name and %d under the new, want exactly one in total", old, new)
+	} else if old != 1 {
+		t.Errorf("the rename committed after the snapshot was taken, so the scan must show the old name")
+	}
+	if _, err := reader.FindEntry(ctx, util.FullPath("/ckpt/x")); err != nil {
+		t.Errorf("once the scan is over the new name must be visible: %v", err)
+	}
+}
+
+// Rows that are not entries, key-value records and a regular file under
+// /buckets with a bucket-like name, are stepped over; a scan rooted at a bucket
+// yields the bucket's own row (default table) and its contents (bucket table).
+func TestScanEntriesRowsAndBucketTables(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	now := time.Now()
+	// The filer creates bucket tables on bucket creation; the store layer alone does not.
+	if err := store.CreateTable(ctx, "realbucket"); err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range []*filer.Entry{
+		{FullPath: "/buckets", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
+		{FullPath: "/buckets/realbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
+		{FullPath: "/buckets/realbucket/f1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
+		{FullPath: "/buckets/readme", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
+		{FullPath: "/topics/t1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
+	} {
+		if err := store.InsertEntry(ctx, e); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, key := range []string{"filer.store.id", "/looks/like/a/path", "x"} {
+		if err := store.KvPut(ctx, []byte(key), []byte("opaque value, not an Entry")); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	collect := func(root string) []string {
+		var got []string
+		if err := store.ScanEntries(ctx, util.FullPath(root), func(_ context.Context, e *filer.Entry) error {
+			got = append(got, string(e.FullPath))
+			return nil
+		}); err != nil {
+			t.Fatalf("scan %s: %v", root, err)
+		}
+		sort.Strings(got)
+		return got
+	}
+	if got, want := collect("/"), []string{"/buckets", "/buckets/readme", "/buckets/realbucket", "/buckets/realbucket/f1", "/topics/t1"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("scan /: got %v, want %v", got, want)
+	}
+	if got, want := collect("/buckets/realbucket"), []string{"/buckets/realbucket", "/buckets/realbucket/f1"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("scan /buckets/realbucket: got %v, want %v", got, want)
+	}
+
+	// A bucket directory whose table does not exist fails the scan: from the
+	// snapshot it cannot be told from a table dropped a moment ago.
+	if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/buckets/ghostbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}); err != nil {
+		t.Fatal(err)
+	}
+	err := store.ScanEntries(ctx, "/", func(context.Context, *filer.Entry) error { return nil })
+	if err == nil || !strings.Contains(err.Error(), "ghostbucket") {
+		t.Fatalf("scan over a bucket without a table must fail naming it, got %v", err)
+	}
+}
+
+// Without WAL the store declines the scan rather than block writers for its
+// whole duration.
+func TestScanEntriesDeclinedWithoutWAL(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
+	if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
+		t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
+	}
+}
+
+// A store whose pool is one connection, and no scan connection of its own,
+// declines rather than hold that connection for the whole scan.
+func TestScanEntriesDeclinedOnSingleConnectionPool(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		t.Fatal(err)
+	}
+	store.ScanDB.Close()
+	store.ScanDB = nil
+	err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
+	if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
+		t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
+	}
+}
+
+// A hard-linked entry's chunks live in the shared key-value record, not in
+// its path row; the wrapper resolves them from inside the snapshot, on the
+// single connection the scan holds.
+func TestScanEntriesResolvesHardLinks(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		t.Fatal(err)
+	}
+	wrapper := filer.NewFilerStoreWrapper(store)
+	ctx := context.Background()
+	now := time.Now()
+	linked := &filer.Entry{
+		FullPath:        "/buckets/hlbucket/f0001",
+		Attr:            filer.Attr{Mode: 0644, Mtime: now, Crtime: now},
+		HardLinkId:      filer.HardLinkId("hardlink-00000001"),
+		HardLinkCounter: 2,
+		Chunks:          []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 4096}},
+	}
+	if err := wrapper.InsertEntry(ctx, linked); err != nil {
+		t.Fatal(err)
+	}
+
+	var got *filer.Entry
+	if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
+		if e.FullPath == linked.FullPath {
+			got = e
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if got == nil || len(got.GetChunks()) != 1 || got.GetChunks()[0].FileId != "3,01637037d6" || got.HardLinkCounter != 2 {
+		t.Fatalf("hard-linked entry not resolved from its key-value record: %+v", got)
+	}
+
+	// Once the record is gone the path row is all there is; the scan hands it
+	// over instead of failing on that entry every time it runs.
+	if err := store.KvDelete(ctx, []byte(linked.HardLinkId)); err != nil {
+		t.Fatal(err)
+	}
+	got = nil
+	if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
+		if e.FullPath == linked.FullPath {
+			got = e
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("a missing hard-link record must not fail the scan: %v", err)
+	}
+	if got == nil {
+		t.Fatal("entry with a missing hard-link record not handed over")
+	}
+}
+
+// The keyset page must be a primary-key seek; an OR-shaped cursor predicate
+// makes SQLite scan the table from the start for every page.
+func TestScanPageQuerySeeksPrimaryKey(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	sqlText, args := store.GetSqlScanPage("filemeta", int64(-1), "", 10)
+	rows, err := store.DB.Query("EXPLAIN QUERY PLAN "+sqlText, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	var plan []string
+	for rows.Next() {
+		var id, parent, notused int
+		var detail string
+		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
+			t.Fatal(err)
+		}
+		plan = append(plan, detail)
+	}
+	joined := strings.Join(plan, " | ")
+	if !strings.Contains(joined, "SEARCH") || strings.Contains(joined, "SCAN") {
+		t.Fatalf("page query does not seek the primary key: %s", joined)
+	}
+}
+
+// A bucket name only the process cache remembers may have its table dropped
+// meanwhile; the scan skips it instead of failing.
+func TestScanEntriesSkipsCachedOnlyBucketTable(t *testing.T) {
+	store := &SqliteStore{}
+	if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
+		t.Fatal(err)
+	}
+	defer store.Shutdown()
+	if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
+		t.Fatal(err)
+	}
+	store.OnBucketCreation("gonebucket") // the table is never created
+	if err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }); err != nil {
+		t.Fatalf("cached-only bucket with a dropped table must not fail the scan: %v", err)
+	}
+}

weed/iam/helpers.go

@@ -98,11 +98,13 @@ var fineGrainedActionMap = map[string]string{
 // prefixed (s3:Action) and unprefixed (Action) variants in init().
 var baseS3ActionMap = map[string]string{
 	// Object operations
-	"DeleteObject":        s3_constants.ACTION_WRITE,
-	"PutObject":           s3_constants.ACTION_WRITE,
-	"GetObject":           s3_constants.ACTION_READ,
-	"DeleteObjectVersion": s3_constants.ACTION_WRITE,
-	"GetObjectVersion":    s3_constants.ACTION_READ,
+	"DeleteObject":               s3_constants.ACTION_WRITE,
+	"PutObject":                  s3_constants.ACTION_WRITE,
+	"GetObject":                  s3_constants.ACTION_READ,
+	"DeleteObjectVersion":        s3_constants.ACTION_WRITE,
+	"GetObjectVersion":           s3_constants.ACTION_READ,
+	"GetObjectAttributes":        s3_constants.ACTION_READ,
+	"GetObjectVersionAttributes": s3_constants.ACTION_READ,
 	// Tagging operations
 	"GetObjectTagging":           s3_constants.ACTION_TAGGING,
 	"GetObjectVersionTagging":    s3_constants.ACTION_TAGGING,
@@ -115,6 +117,7 @@ var baseS3ActionMap = map[string]string{
 	"DeleteBucketTagging":        s3_constants.ACTION_TAGGING,
 	// ACL operations
 	"PutObjectAcl":        s3_constants.ACTION_WRITE_ACP,
+	"PutObjectVersionAcl": s3_constants.ACTION_WRITE_ACP,
 	"GetObjectAcl":        s3_constants.ACTION_READ_ACP,
 	"GetObjectVersionAcl": s3_constants.ACTION_READ_ACP,
 	"PutBucketAcl":        s3_constants.ACTION_WRITE_ACP,

weed/iam/integration/iam_manager.go

@@ -53,7 +53,13 @@ type IAMManager struct {
 	oidcRetryAgain bool
 	// oidcRefreshMu serializes refreshes from reading the store to handing
 	// STS the result, so an older snapshot cannot replace a newer one.
-	oidcRefreshMu        sync.Mutex
+	oidcRefreshMu sync.Mutex
+	// oidcManagedProviders caches providers refreshOIDCProvidersFrom built,
+	// keyed by record identity, so an unchanged record reuses its provider:
+	// rebuilding would drop the provider's consumed-token (jti) replay cache
+	// and let an already-used token be presented again. Guarded by
+	// oidcRefreshMu.
+	oidcManagedProviders map[string]*oidcManagedProvider
 	oidcAuditSink        OIDCProviderAuditSink
 	revocationStore      SessionRevocationStore
 	filerAddressProvider func() string // Function to get current filer address
@@ -943,14 +949,65 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
 		return fmt.Errorf("list OIDC providers: the store was replaced during the refresh")
 	}
 	byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records))
+	if m.oidcManagedProviders == nil {
+		m.oidcManagedProviders = make(map[string]*oidcManagedProvider)
+	}
+	seen := make(map[string]struct{}, len(records))
+	// Records for one issuer with identical validation settings share a single
+	// provider — and with it the consumed-token (jti) replay store — so a token
+	// spent through one account's record stays spent through the other's too.
+	sharedProviders := make(map[string]*oidc.OIDCProvider)
 	for _, rec := range records {
 		if rec == nil || rec.URL == "" {
 			continue
 		}
-		provider, err := buildOIDCProviderFromRecord(rec)
-		if err != nil {
-			glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err)
-			continue
+		key := rec.ARN
+		if key == "" {
+			key = rec.URL + "\x00" + rec.AccountID
+		}
+		seen[key] = struct{}{}
+		fingerprint := oidcProviderFingerprint(rec)
+		configFingerprint := oidcProviderConfigFingerprint(rec)
+		var provider *oidc.OIDCProvider
+		cached := m.oidcManagedProviders[key]
+		if shared, ok := sharedProviders[configFingerprint]; ok {
+			provider = shared
+			if cached != nil && cached.provider != nil && cached.provider != shared {
+				// The record's own provider is superseded; its consumed
+				// tokens move to the shared store so none reopen for replay.
+				shared.MergeReplayStateFrom(cached.provider)
+			}
+			m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
+			if cached != nil && cached.provider != nil && cached.provider != shared && !m.oidcProviderStillReferenced(cached.provider) {
+				if err := cached.provider.Shutdown(ctx); err != nil {
+					glog.Warningf("shutdown superseded OIDC provider %s: %v", rec.ARN, err)
+				}
+			}
+		} else if cached != nil && cached.fingerprint == fingerprint {
+			provider = cached.provider
+			sharedProviders[configFingerprint] = provider
+		} else {
+			var err error
+			provider, err = buildOIDCProviderFromRecord(rec)
+			if err != nil {
+				delete(m.oidcManagedProviders, key)
+				glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err)
+				continue
+			}
+			if cached != nil && cached.provider != nil {
+				// The rebuild must keep rejecting tokens the replaced
+				// provider consumed; carry its replay store over, then stop
+				// its cleanup worker so only the new one sweeps the store —
+				// unless another record still references that provider.
+				provider.AdoptReplayStateFrom(cached.provider)
+			}
+			sharedProviders[configFingerprint] = provider
+			m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
+			if cached != nil && cached.provider != nil && !m.oidcProviderStillReferenced(cached.provider) {
+				if err := cached.provider.Shutdown(ctx); err != nil {
+					glog.Warningf("shutdown replaced OIDC provider %s: %v", rec.ARN, err)
+				}
+			}
 		}
 		// Multiple records may share an issuer when each is scoped to a
 		// different account; STS picks the right one at validation time
@@ -960,10 +1017,71 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
 			Provider:  provider,
 		})
 	}
+	for key, managed := range m.oidcManagedProviders {
+		if _, ok := seen[key]; !ok {
+			delete(m.oidcManagedProviders, key)
+			// A removed record's provider may be shared by records that are
+			// still present; shut it down only when nothing references it.
+			if managed.provider != nil && !m.oidcProviderStillReferenced(managed.provider) {
+				if err := managed.provider.Shutdown(ctx); err != nil {
+					glog.Warningf("shutdown removed OIDC provider %s: %v", key, err)
+				}
+			}
+		}
+	}
 	m.stsService.SetIAMManagedOIDCProviders(byIssuer)
 	return nil
 }
 
+// oidcManagedProvider is a runtime provider together with the fingerprint of
+// the record it was built from.
+type oidcManagedProvider struct {
+	fingerprint string
+	provider    *oidc.OIDCProvider
+}
+
+// oidcProviderStillReferenced reports whether any managed record still points
+// at the provider, so a shared provider is not shut down with a removed record.
+func (m *IAMManager) oidcProviderStillReferenced(provider *oidc.OIDCProvider) bool {
+	for _, managed := range m.oidcManagedProviders {
+		if managed.provider == provider {
+			return true
+		}
+	}
+	return false
+}
+
+// oidcProviderFingerprint identifies the record content a provider was built
+// from, so a refresh rebuilds the provider only when that content changed.
+func oidcProviderFingerprint(rec *OIDCProviderRecord) string {
+	key, _ := json.Marshal(struct {
+		ARN                     string
+		AccountID               string
+		URL                     string
+		ClientIDs               []string
+		Thumbprints             []string
+		AllowedPrincipalTagKeys []string
+		PolicyClaim             string
+	}{rec.ARN, rec.AccountID, rec.URL, rec.ClientIDs, rec.Thumbprints, rec.AllowedPrincipalTagKeys, rec.PolicyClaim})
+	sum := sha256.Sum256(key)
+	return hex.EncodeToString(sum[:])
+}
+
+// oidcProviderConfigFingerprint identifies the validation settings a provider
+// is built from — everything buildOIDCProviderFromRecord consumes except the
+// record's name — so records that differ only in identity reuse one provider.
+func oidcProviderConfigFingerprint(rec *OIDCProviderRecord) string {
+	key, _ := json.Marshal(&oidc.OIDCConfig{
+		Issuer:                  rec.URL,
+		ClientIDs:               rec.ClientIDs,
+		Thumbprints:             rec.Thumbprints,
+		AllowedPrincipalTagKeys: rec.AllowedPrincipalTagKeys,
+		PolicyClaim:             rec.PolicyClaim,
+	})
+	sum := sha256.Sum256(key)
+	return hex.EncodeToString(sum[:])
+}
+
 // buildOIDCProviderFromRecord turns a stored record into a runtime
 // OIDCProvider. The provider name is the ARN so re-registration is
 // idempotent and never collides with static-config entries.

weed/iam/integration/oidc_provider_persist_test.go

@@ -559,3 +559,93 @@ func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
 	}
 	assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
 }
+
+// A refresh must not rebuild a provider whose record is unchanged: rebuilding
+// would drop the provider's consumed-token (jti) replay cache and let an
+// already-used token be presented again.
+func TestRefreshReusesProviderWhileRecordIsUnchanged(t *testing.T) {
+	ctx := context.Background()
+	store := NewMemoryOIDCProviderStore()
+	mgr := startServer(t, store)
+
+	arn := arnOf(t, persistTestAPIIssuer)
+	require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
+		ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
+	}))
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	first := mgr.oidcManagedProviders[arn].provider
+	require.NotNil(t, first)
+
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	assert.Same(t, first, mgr.oidcManagedProviders[arn].provider,
+		"an unchanged record was rebuilt, dropping its replay cache")
+
+	// A changed record must rebuild so the new configuration applies.
+	require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
+		ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud", "aud2"},
+	}))
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	rebuilt := mgr.oidcManagedProviders[arn].provider
+	require.NotNil(t, rebuilt)
+	assert.NotSame(t, first, rebuilt, "a changed record kept the old provider")
+
+	// Deleting the record drops the provider from the cache and the STS map.
+	require.NoError(t, store.DeleteProvider(ctx, "", arn))
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	assert.NotContains(t, mgr.oidcManagedProviders, arn)
+	assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer))
+}
+
+// stsKnowsIssuerForAccount is stsKnowsIssuer through the account-scoped
+// lookup used for providers that have no global record.
+func stsKnowsIssuerForAccount(t *testing.T, mgr *IAMManager, issuer, accountID string) bool {
+	t.Helper()
+	tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+		"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
+	}).SignedString([]byte("not-the-providers-key"))
+	require.NoError(t, err)
+	_, _, err = mgr.GetSTSService().ValidateWebIdentityTokenForAccount(context.Background(), tok, accountID)
+	require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
+	return !strings.Contains(err.Error(), "no identity provider registered")
+}
+
+// Two records for one issuer with identical validation settings share a
+// single provider — and with it the consumed-token (jti) replay store — so a
+// token spent through one account's record stays spent through the other's.
+// Removing one record of the pair must not shut the shared provider down.
+func TestSameIssuerRecordsShareOneProvider(t *testing.T) {
+	ctx := context.Background()
+	store := NewMemoryOIDCProviderStore()
+	mgr := startServer(t, store)
+
+	arnA := arnOf(t, persistTestAPIIssuer) + "-account-a"
+	arnB := arnOf(t, persistTestAPIIssuer) + "-account-b"
+	for _, rec := range []*OIDCProviderRecord{
+		{ARN: arnA, URL: persistTestAPIIssuer, AccountID: "111111111111", ClientIDs: []string{"aud"}},
+		{ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"aud"}},
+	} {
+		require.NoError(t, store.StoreProvider(ctx, "", rec))
+	}
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+
+	shared := mgr.oidcManagedProviders[arnA].provider
+	require.NotNil(t, shared)
+	assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
+		"identical records for one issuer built separate replay stores")
+
+	// Removing one record of the pair keeps the shared provider alive.
+	require.NoError(t, store.DeleteProvider(ctx, "", arnA))
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
+		"a removed sibling record shut down the shared provider")
+	assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
+
+	// The remaining record diverging rebuilds its own provider.
+	require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
+		ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"b-aud"},
+	}))
+	require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
+	assert.NotSame(t, shared, mgr.oidcManagedProviders[arnB].provider,
+		"a changed record kept the shared provider")
+	assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
+}

weed/iam/oidc/mock_provider.go

@@ -75,6 +75,17 @@ func (m *MockOIDCProvider) Authenticate(ctx context.Context, token string) (*pro
 	return identity, nil
 }
 
+// AuthenticateSingleUse stays on the mock path: the embedded provider's
+// method would bypass the mock's test data and demand a real JWKS.
+func (m *MockOIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
+	return m.Authenticate(ctx, token)
+}
+
+// ValidateTokenSingleUse stays on the mock path; see AuthenticateSingleUse.
+func (m *MockOIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
+	return m.ValidateToken(ctx, token)
+}
+
 // ValidateToken validates tokens using test data
 func (m *MockOIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
 	if !m.initialized {

weed/iam/oidc/oidc_provider.go

@@ -6,6 +6,7 @@ import (
 	"crypto/elliptic"
 	"crypto/rsa"
 	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
@@ -19,6 +20,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/golang-jwt/jwt/v5"
@@ -27,7 +29,44 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 )
 
-// OIDCProvider implements OpenID Connect authentication
+const (
+	// clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519)
+	clockSkewTolerance = 5 * time.Minute
+	// defaultJTICacheSize is the default maximum number of JTI entries to cache
+	defaultJTICacheSize = 10000
+	// defaultCleanupInterval is how often to scan for expired JTI entries
+	defaultCleanupInterval = 5 * time.Minute
+	// eagerCleanupMinInterval rate limits opportunistic sweeps near capacity
+	eagerCleanupMinInterval = 5 * time.Second
+	// tokenFingerprintPrefix marks replay-cache keys derived from the token's
+	// own fingerprint, used when the token carries no usable jti claim
+	tokenFingerprintPrefix = "fp:"
+)
+
+// jtiEntry represents a cached JTI with metadata
+type jtiEntry struct {
+	subject   string
+	expiresAt time.Time
+	// uncounted marks an entry merged in past the capacity bound: it keeps
+	// rejecting replays but does not charge jtiCount, so merged excess can
+	// neither reject new exchanges nor be double-counted when swept.
+	uncounted bool
+}
+
+// OIDCProvider implements OpenID Connect authentication with JTI replay protection.
+//
+// Replay Protection:
+// The provider validates the 'jti' (JWT ID) claim to prevent token replay attacks.
+// Each JTI is stored in memory until the token expires. If a token with a duplicate
+// JTI is presented, it is rejected with ErrProviderTokenReplayed.
+//
+// The JTI cache is bounded by JTICacheMaxSize and uses automatic TTL-based expiration.
+// A token without a usable jti claim is tracked by the token's own fingerprint
+// instead, so a repeated presentation is still detected as a replay.
+//
+// Thread Safety:
+// All methods are safe for concurrent use. The JTI cache uses atomic LoadOrStore
+// operations to prevent TOCTOU race conditions.
 type OIDCProvider struct {
 	name        string
 	config      *OIDCConfig
@@ -44,6 +83,32 @@ type OIDCProvider struct {
 	jwksFetchedAt   time.Time
 	resolvedJWKSUri string
 	discoveryFailed bool
+
+	// JTI replay protection fields. jtiStore and jtiCount are allocated once
+	// in NewOIDCProvider and survive re-initialization, so a config refresh
+	// cannot reopen already-consumed tokens for replay.
+	jtiEnabled   atomic.Bool
+	jtiStore     *sync.Map     // map[string]*jtiEntry - atomic storage for consumed token keys
+	jtiCount     *atomic.Int64 // entries in jtiStore; shared with the store
+	jtiMaxSize   atomic.Int64
+	jtiLastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups
+
+	// jtiStoreMu guards the jtiStore/jtiCount pair: replay-state handoffs
+	// (AdoptReplayStateFrom, MergeReplayStateFrom) repoint them under the
+	// write lock while the consume path holds a read lock, so a consume can
+	// never land in a store the handoff already drained and abandoned.
+	// It is always acquired before jtiSweepMu, never while holding it.
+	jtiStoreMu sync.RWMutex
+
+	// jtiMu guards the lifecycle of the cleanup worker: jtiCleanupCancel and
+	// jtiCleanupDone. The worker itself only touches jtiStore/jtiCount, so
+	// holding jtiMu while waiting for it is safe.
+	jtiMu            sync.Mutex
+	jtiCleanupCancel context.CancelFunc
+	jtiCleanupDone   chan struct{} // Closed when cleanup goroutine exits
+
+	// jtiSweepMu serializes cache sweeps so at most one scan runs at a time.
+	jtiSweepMu sync.Mutex
 }
 
 // OIDCConfig holds OIDC provider configuration
@@ -107,6 +172,13 @@ type OIDCConfig struct {
 	// TLSInsecureSkipVerify controls whether to skip TLS verification.
 	// WARNING: Should only be used in development/testing environments. Never use in production.
 	TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty"`
+
+	// JTIReplayProtectionEnabled enables token replay protection via JTI tracking (default: true)
+	// Use pointer to distinguish between "not set" (nil = default true) and "explicitly false"
+	JTIReplayProtectionEnabled *bool `json:"jtiReplayProtectionEnabled,omitempty"`
+
+	// JTICacheMaxSize sets maximum number of JTI entries to cache (default: 10000)
+	JTICacheMaxSize int `json:"jtiCacheMaxSize,omitempty"`
 }
 
 // PrincipalTagsClaim is the AWS-defined namespace claim that carries
@@ -319,6 +391,8 @@ func NewOIDCProvider(name string) *OIDCProvider {
 	return &OIDCProvider{
 		name:       name,
 		httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect},
+		jtiStore:   &sync.Map{},
+		jtiCount:   &atomic.Int64{},
 	}
 }
 
@@ -429,10 +503,62 @@ func (p *OIDCProvider) Initialize(config interface{}) error {
 		CheckRedirect: refuseDowngradeRedirect,
 	}
 
+	// Configure JTI replay protection last, so a config failure above cannot
+	// leave a running worker behind. A previous worker is always stopped,
+	// whether the new configuration enables protection or disables it.
+	// The consumed-token store survives re-initialization: resetting it would
+	// let already-seen tokens be replayed against the refreshed provider.
+	jtiEnabled := true
+	if oidcConfig.JTIReplayProtectionEnabled != nil {
+		jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled
+	}
+
+	p.jtiMu.Lock()
+	p.stopJTICleanupWorkerLocked()
+	if jtiEnabled {
+		maxSize := int64(oidcConfig.JTICacheMaxSize)
+		if maxSize <= 0 {
+			maxSize = defaultJTICacheSize
+		}
+		// the bound must land before the flag: a consume that sees enabled
+		// with a zero bound would reject the first token it registers
+		p.jtiMaxSize.Store(maxSize)
+		ctx, cancel := context.WithCancel(context.Background())
+		done := make(chan struct{})
+		p.jtiCleanupCancel = cancel
+		p.jtiCleanupDone = done
+		p.jtiEnabled.Store(true)
+		p.jtiMu.Unlock()
+
+		go p.cleanupExpiredJTIs(ctx, done, defaultCleanupInterval)
+		glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", p.name, maxSize)
+	} else {
+		p.jtiEnabled.Store(false)
+		p.jtiMu.Unlock()
+		glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", p.name)
+	}
+
 	// For testing, we'll skip the actual OIDC client initialization
 	return nil
 }
 
+// stopJTICleanupWorkerLocked cancels the current cleanup goroutine and waits
+// briefly for it to exit. Caller must hold jtiMu.
+func (p *OIDCProvider) stopJTICleanupWorkerLocked() {
+	if p.jtiCleanupCancel == nil {
+		return
+	}
+	p.jtiCleanupCancel()
+	select {
+	case <-p.jtiCleanupDone:
+		glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", p.name)
+	case <-time.After(2 * time.Second):
+		glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", p.name)
+	}
+	p.jtiCleanupCancel = nil
+	p.jtiCleanupDone = nil
+}
+
 // validateConfig validates the OIDC configuration
 func (p *OIDCProvider) validateConfig(config *OIDCConfig) error {
 	if config.Issuer == "" {
@@ -479,6 +605,18 @@ func (p *OIDCProvider) allowedAudiences() []string {
 
 // Authenticate authenticates a user with an OIDC token
 func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
+	return p.authenticate(ctx, token, false)
+}
+
+// AuthenticateSingleUse authenticates and consumes the token so a second
+// presentation is rejected as a replay. The web-identity exchange uses it,
+// where a token buys a session exactly once; callers that validate the same
+// token on every request use Authenticate.
+func (p *OIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
+	return p.authenticate(ctx, token, true)
+}
+
+func (p *OIDCProvider) authenticate(ctx context.Context, token string, singleUse bool) (*providers.ExternalIdentity, error) {
 	if !p.initialized {
 		return nil, fmt.Errorf("provider not initialized")
 	}
@@ -488,7 +626,7 @@ func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*provide
 	}
 
 	// Validate token and get claims
-	claims, err := p.ValidateToken(ctx, token)
+	claims, err := p.validateToken(ctx, token, singleUse)
 	if err != nil {
 		return nil, err
 	}
@@ -655,6 +793,16 @@ func (p *OIDCProvider) getUserInfoWithToken(ctx context.Context, userID, accessT
 
 // ValidateToken validates an OIDC JWT token
 func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
+	return p.validateToken(ctx, token, false)
+}
+
+// ValidateTokenSingleUse validates and consumes the token: a second
+// presentation of it is rejected as a replay. See AuthenticateSingleUse.
+func (p *OIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
+	return p.validateToken(ctx, token, true)
+}
+
+func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUse bool) (*providers.TokenClaims, error) {
 	if !p.initialized {
 		return nil, fmt.Errorf("provider not initialized")
 	}
@@ -787,6 +935,87 @@ func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*provid
 		tokenClaims.Claims[key] = value
 	}
 
+	// Extract and validate JTI claim for replay protection. The check runs
+	// only for single-use callers: a bearer token is meant to be presented on
+	// every request, so validating it must not consume it.
+	if singleUse && p.jtiEnabled.Load() {
+		jti, _ := claims["jti"].(string)
+		key := jti
+		if key == "" {
+			// Providers that issue no jti cannot be tracked by claim, so the
+			// signed input keys the entry: a repeated presentation of the same
+			// claims is a replay even when the signature bytes differ (e.g.
+			// ECDSA s/N-s malleability).
+			signingInput := token
+			if i := strings.LastIndexByte(token, '.'); i > 0 {
+				signingInput = token[:i]
+			}
+			sum := sha256.Sum256([]byte(signingInput))
+			key = tokenFingerprintPrefix + hex.EncodeToString(sum[:])
+		}
+
+		expiresAt := tokenClaims.ExpiresAt
+		if expiresAt.IsZero() {
+			// a cache slot is freed only by expiry, so a token without exp
+			// would hold it forever and eventually exhaust the bound
+			return nil, fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims)
+		}
+		expiresAt = expiresAt.Add(clockSkewTolerance)
+		if !time.Now().Before(expiresAt) {
+			return nil, fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired)
+		}
+
+		entry := &jtiEntry{
+			subject:   subject,
+			expiresAt: expiresAt,
+		}
+
+		// The read lock pins the store/count pair for the whole test-and-set
+		// so a replay-state handoff cannot repoint them mid-check and strand
+		// this consume in an abandoned store.
+		p.jtiStoreMu.RLock()
+		store, count := p.jtiStore, p.jtiCount
+
+		// Atomic test-and-set detects replays before any capacity work runs,
+		// so a replayed token cannot trigger cleanup scans.
+		if _, loaded := store.LoadOrStore(key, entry); loaded {
+			p.jtiStoreMu.RUnlock()
+			glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)",
+				p.name, jtiLogKey(key))
+			return nil, fmt.Errorf("%w: jti %s has already been used",
+				providers.ErrProviderTokenReplayed, jtiLogKey(key))
+		}
+
+		// The count is incremented as part of the insert, so concurrent
+		// validations cannot all slip under the limit: whoever pushes the
+		// count over evicts their own entry and is rejected.
+		newCount := count.Add(1)
+		maxSize := p.jtiMaxSize.Load()
+		if newCount > maxSize {
+			// Over the bound: expired entries may still occupy slots, so
+			// reclaim them with a synchronized sweep before rejecting.
+			p.sweepJTIStore(store, count)
+			if count.Load() > maxSize {
+				if _, loaded := store.LoadAndDelete(key); loaded {
+					count.Add(-1)
+				}
+				p.jtiStoreMu.RUnlock()
+				glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+
+					"This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.",
+					p.name, maxSize)
+				return nil, fmt.Errorf("JTI cache capacity exceeded")
+			}
+		} else if newCount >= maxSize*9/10 {
+			glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup",
+				p.name, (newCount*100)/maxSize, newCount, maxSize)
+			go p.performEagerCleanup()
+		}
+		p.jtiStoreMu.RUnlock()
+
+		glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)",
+			p.name, jtiLogKey(key), expiresAt, newCount)
+	}
+
 	return tokenClaims, nil
 }
 
@@ -1215,3 +1444,195 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p
 
 	return identity
 }
+
+// evictExpiredJTIs removes entries whose expiry has passed and returns how
+// many it deleted. Caller must hold jtiSweepMu.
+func (p *OIDCProvider) evictExpiredJTIs(now time.Time, store *sync.Map, count *atomic.Int64) int {
+	removed := 0
+	store.Range(func(key, value interface{}) bool {
+		expiresAt := value.(*jtiEntry).expiresAt
+		if !expiresAt.IsZero() && now.After(expiresAt) {
+			// LoadAndDelete reports whether this call actually removed the
+			// entry, so concurrent deletes cannot decrement the count twice.
+			if v, loaded := store.LoadAndDelete(key); loaded {
+				if !v.(*jtiEntry).uncounted {
+					count.Add(-1)
+				}
+				removed++
+			}
+		}
+		return true
+	})
+	return removed
+}
+
+// replayState returns the provider's current store/count pair.
+func (p *OIDCProvider) replayState() (*sync.Map, *atomic.Int64) {
+	p.jtiStoreMu.RLock()
+	defer p.jtiStoreMu.RUnlock()
+	return p.jtiStore, p.jtiCount
+}
+
+// sweepJTIStore runs one serialized expiry scan over the given store. It only
+// takes jtiSweepMu, so callers already holding jtiStoreMu may use it.
+func (p *OIDCProvider) sweepJTIStore(store *sync.Map, count *atomic.Int64) {
+	p.jtiSweepMu.Lock()
+	defer p.jtiSweepMu.Unlock()
+	p.jtiLastSweep.Store(time.Now().UnixNano())
+	if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 {
+		glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)",
+			p.name, removed, count.Load())
+	}
+}
+
+// sweepExpiredJTIs runs a scan that is guaranteed to complete before the
+// caller proceeds; the capacity-rejection path uses it so a fresh token is
+// not refused while expired entries still hold slots.
+func (p *OIDCProvider) sweepExpiredJTIs() {
+	store, count := p.replayState()
+	p.sweepJTIStore(store, count)
+}
+
+// performEagerCleanup performs an opportunistic cleanup of expired JTI
+// entries when the cache approaches capacity. At most one sweep runs at a
+// time and no more often than eagerCleanupMinInterval, so a burst of
+// requests near the limit cannot each turn into a full-cache scan.
+func (p *OIDCProvider) performEagerCleanup() {
+	if !p.jtiEnabled.Load() {
+		return
+	}
+	if time.Since(time.Unix(0, p.jtiLastSweep.Load())) < eagerCleanupMinInterval {
+		return
+	}
+	store, count := p.replayState()
+	if !p.jtiSweepMu.TryLock() {
+		return
+	}
+	defer p.jtiSweepMu.Unlock()
+	p.jtiLastSweep.Store(time.Now().UnixNano())
+
+	beforeCount := count.Load()
+	if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 {
+		glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)",
+			p.name, removed, beforeCount, count.Load())
+	} else {
+		glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+
+			"Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.",
+			p.name)
+	}
+}
+
+// cleanupExpiredJTIs periodically removes expired JTI entries from the cache.
+// This prevents memory growth by removing JTIs after their tokens have expired.
+// Runs in a background goroutine until ctx is cancelled; done is closed on exit.
+func (p *OIDCProvider) cleanupExpiredJTIs(ctx context.Context, done chan struct{}, interval time.Duration) {
+	defer close(done)
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", p.name, interval)
+
+	for {
+		select {
+		case <-ctx.Done():
+			glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", p.name)
+			return
+		case <-ticker.C:
+			p.sweepExpiredJTIs()
+		}
+	}
+}
+
+// Shutdown gracefully stops the JTI cleanup goroutine.
+// Should be called when the provider is being shut down to prevent goroutine leaks.
+func (p *OIDCProvider) Shutdown(ctx context.Context) error {
+	p.jtiMu.Lock()
+	cancel, done := p.jtiCleanupCancel, p.jtiCleanupDone
+	p.jtiMu.Unlock()
+
+	if cancel == nil {
+		return nil
+	}
+	cancel()
+
+	select {
+	case <-done:
+		glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", p.name)
+	case <-ctx.Done():
+		glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", p.name, ctx.Err())
+		return ctx.Err()
+	}
+
+	p.jtiMu.Lock()
+	if p.jtiCleanupDone == done {
+		p.jtiCleanupCancel = nil
+		p.jtiCleanupDone = nil
+	}
+	p.jtiMu.Unlock()
+	return nil
+}
+
+// AdoptReplayStateFrom takes over the consumed-token store of the provider it
+// replaces, so tokens it already saw stay rejected across the rebuild. The
+// caller must shut down the replaced provider; only one cleanup worker may
+// sweep the shared store at a time. Call before the new provider serves
+// validators; jtiSweepMu excludes its own cleanup worker during the swap.
+func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) {
+	if old == nil || old == p {
+		return
+	}
+	p.jtiStoreMu.Lock()
+	p.jtiSweepMu.Lock()
+	p.jtiStore, p.jtiCount = old.jtiStore, old.jtiCount
+	p.jtiSweepMu.Unlock()
+	p.jtiStoreMu.Unlock()
+}
+
+// MergeReplayStateFrom hands this provider's store to another provider and
+// then folds the other's consumed-token entries in, so rejections the other
+// recorded keep holding when its records move to a shared provider. Repointing
+// first means tokens the other still consumes — in-flight exchanges or ones
+// routed to it before the provider map updates — land in the shared store
+// instead of a store this merge already drained. Unlike AdoptReplayStateFrom
+// it swaps nothing of its own, so it is safe on a provider that is already
+// serving validators.
+func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) {
+	if other == nil || other == p {
+		return
+	}
+	p.jtiStoreMu.RLock()
+	defer p.jtiStoreMu.RUnlock()
+	store, count := p.jtiStore, p.jtiCount
+
+	other.jtiStoreMu.Lock()
+	oldStore := other.jtiStore
+	if oldStore == store {
+		other.jtiStoreMu.Unlock()
+		return
+	}
+	other.jtiStore, other.jtiCount = store, count
+	other.jtiStoreMu.Unlock()
+
+	p.jtiSweepMu.Lock()
+	defer p.jtiSweepMu.Unlock()
+	now := time.Now()
+	oldStore.Range(func(key, value interface{}) bool {
+		entry := value.(*jtiEntry)
+		if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) {
+			return true
+		}
+		// A consumed token is never dropped to fit the bound: merged entries
+		// stay but do not charge the count, so a store filled past the bound
+		// by the merge cannot reject new exchanges while it drains by expiry.
+		store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true})
+		return true
+	})
+}
+
+// jtiLogKey redacts a token identifier for logs and error strings: a stable
+// fingerprint that still lets operators correlate repeated presentations.
+func jtiLogKey(jti string) string {
+	sum := sha256.Sum256([]byte(jti))
+	return hex.EncodeToString(sum[:8])
+}

weed/iam/providers/errors.go

@@ -19,4 +19,7 @@ var (
 
 	// ErrProviderMissingClaims indicates that required claims are missing from the token
 	ErrProviderMissingClaims = errors.New("provider: missing required claims")
+
+	// ErrProviderTokenReplayed is returned when a token with duplicate JTI is detected
+	ErrProviderTokenReplayed = errors.New("provider: token has been replayed")
 )

weed/iam/sts/constants.go

@@ -42,21 +42,23 @@ const (
 
 // Configuration Field Names
 const (
-	ConfigFieldFilerAddress          = "filerAddress"
-	ConfigFieldBasePath              = "basePath"
-	ConfigFieldIssuer                = "issuer"
-	ConfigFieldClientID              = "clientId"
-	ConfigFieldClientIDs             = "clientIds"
-	ConfigFieldThumbprints           = "thumbprints"
-	ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys"
-	ConfigFieldPolicyClaim             = "policyClaim"
-	ConfigFieldClientSecret          = "clientSecret"
-	ConfigFieldJWKSUri               = "jwksUri"
-	ConfigFieldScopes                = "scopes"
-	ConfigFieldUserInfoUri           = "userInfoUri"
-	ConfigFieldRedirectUri           = "redirectUri"
-	ConfigFieldTLSCACert             = "tlsCaCert"
-	ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify"
+	ConfigFieldFilerAddress               = "filerAddress"
+	ConfigFieldBasePath                   = "basePath"
+	ConfigFieldIssuer                     = "issuer"
+	ConfigFieldClientID                   = "clientId"
+	ConfigFieldClientIDs                  = "clientIds"
+	ConfigFieldThumbprints                = "thumbprints"
+	ConfigFieldAllowedPrincipalTagKeys    = "allowedPrincipalTagKeys"
+	ConfigFieldPolicyClaim                = "policyClaim"
+	ConfigFieldClientSecret               = "clientSecret"
+	ConfigFieldJWKSUri                    = "jwksUri"
+	ConfigFieldScopes                     = "scopes"
+	ConfigFieldUserInfoUri                = "userInfoUri"
+	ConfigFieldRedirectUri                = "redirectUri"
+	ConfigFieldTLSCACert                  = "tlsCaCert"
+	ConfigFieldTLSInsecureSkipVerify      = "tlsInsecureSkipVerify"
+	ConfigFieldJTIReplayProtectionEnabled = "jtiReplayProtectionEnabled"
+	ConfigFieldJTICacheMaxSize            = "jtiCacheMaxSize"
 )
 
 // Error Messages

weed/iam/sts/oidc_provider_lookup_test.go

@@ -11,8 +11,8 @@ import (
 // lookupOIDCProviderForAccount; the lookup never calls Authenticate.
 type stubIdentityProvider struct{ name string }
 
-func (s *stubIdentityProvider) Name() string                       { return s.name }
-func (s *stubIdentityProvider) Initialize(interface{}) error       { return nil }
+func (s *stubIdentityProvider) Name() string                 { return s.name }
+func (s *stubIdentityProvider) Initialize(interface{}) error { return nil }
 func (s *stubIdentityProvider) Authenticate(context.Context, string) (*providers.ExternalIdentity, error) {
 	return nil, nil
 }
@@ -103,3 +103,51 @@ func TestLookupOIDCProviderForAccountFallsBackToStatic(t *testing.T) {
 		t.Fatalf("expected static-config provider as last resort, got %v ok=%v", got, ok)
 	}
 }
+
+// shutdownTrackingProvider records Shutdown calls so tests can observe which
+// replaced providers the swap released.
+type shutdownTrackingProvider struct {
+	stubIdentityProvider
+	shutdownCalls int32
+}
+
+func (s *shutdownTrackingProvider) Shutdown(context.Context) error {
+	s.shutdownCalls++
+	return nil
+}
+
+// A provider dropped from the managed map owns a background JTI cleanup
+// worker; replacing the map must shut it down, while providers that stay in
+// the map keep running.
+func TestSetIAMManagedOIDCProvidersShutsDownDroppedProviders(t *testing.T) {
+	dropped := &shutdownTrackingProvider{stubIdentityProvider{name: "dropped"}, 0}
+	kept := &shutdownTrackingProvider{stubIdentityProvider{name: "kept"}, 0}
+	added := &shutdownTrackingProvider{stubIdentityProvider{name: "added"}, 0}
+
+	s := &STSService{issuerToProvider: map[string]providers.IdentityProvider{}}
+	s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
+		"https://a.example.com": {{Provider: dropped}},
+		"https://b.example.com": {{Provider: kept}, {AccountID: "111111111111", Provider: kept}},
+	})
+
+	s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
+		"https://b.example.com": {{Provider: kept}},
+		"https://c.example.com": {{Provider: added}},
+	})
+
+	if dropped.shutdownCalls != 1 {
+		t.Fatalf("dropped provider was shut down %d times, want 1", dropped.shutdownCalls)
+	}
+	if kept.shutdownCalls != 0 {
+		t.Fatalf("kept provider was shut down %d times, want 0", kept.shutdownCalls)
+	}
+	if added.shutdownCalls != 0 {
+		t.Fatalf("newly added provider was shut down %d times, want 0", added.shutdownCalls)
+	}
+
+	// Clearing the map releases the remaining providers.
+	s.SetIAMManagedOIDCProviders(nil)
+	if kept.shutdownCalls != 1 || added.shutdownCalls != 1 {
+		t.Fatalf("clearing the map shut down kept=%d added=%d, want 1 each", kept.shutdownCalls, added.shutdownCalls)
+	}
+}

weed/iam/sts/provider_factory.go

@@ -1,6 +1,7 @@
 package sts
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -154,6 +155,21 @@ func (f *ProviderFactory) convertToOIDCConfig(configMap map[string]interface{})
 		config.TLSInsecureSkipVerify = tlsInsecureSkipVerify
 	}
 
+	if enabled, ok := configMap[ConfigFieldJTIReplayProtectionEnabled].(bool); ok {
+		config.JTIReplayProtectionEnabled = &enabled
+	}
+
+	switch maxSize := configMap[ConfigFieldJTICacheMaxSize].(type) {
+	case float64:
+		config.JTICacheMaxSize = int(maxSize)
+	case int:
+		config.JTICacheMaxSize = maxSize
+	case json.Number:
+		if n, err := maxSize.Int64(); err == nil {
+			config.JTICacheMaxSize = int(n)
+		}
+	}
+
 	// Convert claims mapping
 	if claimsMapInterface, ok := configMap["claimsMapping"]; ok {
 		claimsMap, err := f.convertToStringMap(claimsMapInterface)

weed/iam/sts/provider_factory_jti_test.go

@@ -0,0 +1,31 @@
+package sts
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertToOIDCConfigCarriesJTISettings(t *testing.T) {
+	factory := NewProviderFactory()
+
+	config, err := factory.convertToOIDCConfig(map[string]interface{}{
+		"issuer":                     "https://issuer.example",
+		"clientId":                   "client",
+		"jtiReplayProtectionEnabled": false,
+		"jtiCacheMaxSize":            float64(42),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, config.JTIReplayProtectionEnabled)
+	assert.False(t, *config.JTIReplayProtectionEnabled)
+	assert.Equal(t, 42, config.JTICacheMaxSize)
+
+	config, err = factory.convertToOIDCConfig(map[string]interface{}{
+		"issuer":   "https://issuer.example",
+		"clientId": "client",
+	})
+	require.NoError(t, err)
+	assert.Nil(t, config.JTIReplayProtectionEnabled)
+	assert.Zero(t, config.JTICacheMaxSize)
+}

weed/iam/sts/session_claims.go

@@ -100,6 +100,11 @@ type STSSessionClaims struct {
 	// guaranteed stable per OpenID Connect Core 1.0. Empty for non-federated
 	// session types.
 	ParentUser string `json:"puid,omitempty"`
+
+	// SourceAccessKeyId is the access key the caller authenticated with when
+	// a SeaweedFS service minted the session for it (AssumeRoleForPrincipal);
+	// empty for every other session type.
+	SourceAccessKeyId string `json:"src_ak,omitempty"`
 }
 
 // NewSTSSessionClaims creates new STS session claims with all required information
@@ -156,6 +161,8 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI
 		ProviderIssuer:   c.ProviderIssuer,
 		RequestContext:   c.RequestContext,
 		ParentUser:       c.ParentUser,
+		// The access key the session was minted for, if recorded
+		SourceAccessKeyId: c.SourceAccessKeyId,
 		// Provide the Subject (sub) from registered claims
 		Subject:     c.Subject,
 		Credentials: credentials,
@@ -250,6 +257,12 @@ func (c *STSSessionClaims) WithSessionName(sessionName string) *STSSessionClaims
 	return c
 }
 
+// WithSourceAccessKeyId records the access key the session was minted for.
+func (c *STSSessionClaims) WithSourceAccessKeyId(accessKeyId string) *STSSessionClaims {
+	c.SourceAccessKeyId = accessKeyId
+	return c
+}
+
 // WithParentUser sets the stable per-identity hash for the session. See
 // ComputeParentUser for the derivation rule.
 func (c *STSSessionClaims) WithParentUser(parentUser string) *STSSessionClaims {

weed/iam/sts/source_access_key_test.go

@@ -0,0 +1,42 @@
+package sts
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newSourceKeyTestSTS(t *testing.T) *STSService {
+	t.Helper()
+	s := NewSTSService()
+	require.NoError(t, s.Initialize(&STSConfig{
+		TokenDuration:    FlexibleDuration{Duration: time.Hour},
+		MaxSessionLength: FlexibleDuration{Duration: 12 * time.Hour},
+		Issuer:           "test-issuer",
+		SigningKey:       []byte("test-signing-key-at-least-32-bytes-long-for-security"),
+	}))
+	return s
+}
+
+// A session minted for a caller's access key carries that key back out of its
+// token; a session minted without one carries none.
+func TestSessionCarriesItsSourceAccessKey(t *testing.T) {
+	s := newSourceKeyTestSTS(t)
+	resp, err := s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t-kabc", "", nil,
+		"iceberg-catalog", "alice", func(c *STSSessionClaims) { c.WithSourceAccessKeyId("AKIASOURCE") })
+	require.NoError(t, err)
+	info, err := s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken)
+	require.NoError(t, err)
+	assert.Equal(t, "AKIASOURCE", info.SourceAccessKeyId)
+	assert.Equal(t, "alice", info.ExternalUserId)
+	assert.Equal(t, "iceberg-catalog", info.IdentityProvider)
+
+	resp, err = s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t", "", nil, "iceberg-catalog", "alice")
+	require.NoError(t, err)
+	info, err = s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken)
+	require.NoError(t, err)
+	assert.Equal(t, "", info.SourceAccessKeyId)
+}

weed/iam/sts/sts_service.go

@@ -316,6 +316,10 @@ type SessionInfo struct {
 	// RoleId is the unique ID of the role the session was issued for; empty
 	// for sessions issued before role IDs were recorded.
 	RoleId string `json:"roleId,omitempty"`
+
+	// SourceAccessKeyId is the access key the session was minted for, when the
+	// minting service recorded it (AssumeRoleForPrincipal).
+	SourceAccessKeyId string `json:"sourceAccessKeyId,omitempty"`
 }
 
 // NewSTSService creates a new STS service
@@ -508,8 +512,35 @@ func (s *STSService) SetIAMManagedOIDCProviders(byIssuer map[string][]ScopedOIDC
 		}
 	}
 	s.iamManagedOIDCMu.Lock()
+	old := s.iamManagedOIDCByIssuer
 	s.iamManagedOIDCByIssuer = cp
 	s.iamManagedOIDCMu.Unlock()
+
+	// Providers dropped by the swap may own background workers (the JTI
+	// cleanup goroutine); stop them so a replaced provider does not leak.
+	kept := make(map[providers.IdentityProvider]struct{})
+	for _, scoped := range cp {
+		for _, sp := range scoped {
+			kept[sp.Provider] = struct{}{}
+		}
+	}
+	stopped := make(map[providers.IdentityProvider]struct{})
+	for _, scoped := range old {
+		for _, sp := range scoped {
+			if _, ok := kept[sp.Provider]; ok {
+				continue
+			}
+			if _, ok := stopped[sp.Provider]; ok {
+				continue
+			}
+			stopped[sp.Provider] = struct{}{}
+			if sd, ok := sp.Provider.(interface{ Shutdown(context.Context) error }); ok {
+				if err := sd.Shutdown(context.Background()); err != nil {
+					glog.Warningf("OIDC provider %q shutdown after replacement: %v", sp.Provider.Name(), err)
+				}
+			}
+		}
+	}
 }
 
 // lookupOIDCProviderForAccount returns the provider that should validate
@@ -583,7 +614,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
 	// caller passed a legacy or claim-based ARN, in which case lookup falls
 	// back to a global (account-less) record only.
 	roleAccountID := utils.ParseRoleARN(request.RoleArn).AccountID
-	externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID)
+	externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID, true)
 	if err != nil {
 		return nil, fmt.Errorf("failed to validate web identity token: %w", err)
 	}
@@ -780,7 +811,7 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
 // issueSession mints temporary credentials and the self-contained JWT that
 // carries the whole session, shared by every assume-role entry point.
 func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string,
-	durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) {
+	durationSeconds *int64, providerName, subject string, options ...func(*STSSessionClaims)) (*AssumeRoleResponse, error) {
 
 	sessionDuration := s.CalculateSessionDuration(durationSeconds)
 	expiresAt := time.Now().Add(sessionDuration)
@@ -810,6 +841,9 @@ func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolic
 	if sessionPolicy != "" {
 		sessionClaims.WithSessionPolicy(sessionPolicy)
 	}
+	for _, option := range options {
+		option(sessionClaims)
+	}
 
 	jwtToken, err := s.tokenGenerator.GenerateJWTWithClaims(sessionClaims)
 	if err != nil {
@@ -863,6 +897,10 @@ type AssumeRoleForPrincipalRequest struct {
 
 	// DurationSeconds requests a session lifetime.
 	DurationSeconds *int64
+
+	// SourceAccessKeyId optionally records the access key the caller
+	// authenticated with, so the session can be tied to that key.
+	SourceAccessKeyId string
 }
 
 // AssumeRoleForPrincipal issues session credentials for a caller that a
@@ -908,7 +946,8 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume
 	// Iceberg credential vending resolves no role definition, so its sessions
 	// carry no role ID and stay bound by name.
 	return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy,
-		request.DurationSeconds, request.ProviderName, request.Principal)
+		request.DurationSeconds, request.ProviderName, request.Principal,
+		func(c *STSSessionClaims) { c.WithSourceAccessKeyId(request.SourceAccessKeyId) })
 }
 
 // ValidateSessionToken validates a session token and returns session information
@@ -969,7 +1008,7 @@ func (s *STSService) validateAssumeRoleWithWebIdentityRequest(request *AssumeRol
 // validateWebIdentityToken validates the web identity token with strict issuer-to-provider mapping
 // SECURITY: JWT tokens with a specific issuer claim MUST only be validated by the provider for that issuer
 // SECURITY: This method only accepts JWT tokens. Non-JWT authentication must use AssumeRoleWithCredentials with explicit ProviderName.
-func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
+func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string, singleUse bool) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
 	// Try to extract issuer from JWT token for strict validation
 	issuer, err := s.extractIssuerFromJWT(token)
 	if err != nil {
@@ -992,8 +1031,10 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
 		return nil, nil, fmt.Errorf("no identity provider registered for issuer: %s", issuer)
 	}
 
-	// Authenticate with the correct provider for this issuer
-	identity, err := provider.Authenticate(ctx, token)
+	// Authenticate with the correct provider for this issuer. Only the STS
+	// exchange authenticates single-use: a bearer token is presented on every
+	// request, so consuming it here would reject all but the first one.
+	identity, err := authenticateOIDCProvider(ctx, provider, token, singleUse)
 	if err != nil {
 		// Map provider errors to STS errors using errors.Is() for robust error checking
 		// This eliminates fragile string matching and provides reliable error classification
@@ -1025,14 +1066,29 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
 // only, then static-config); call ValidateWebIdentityTokenForAccount when the assumed-role account
 // is known.
 func (s *STSService) ValidateWebIdentityToken(ctx context.Context, token string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
-	return s.validateWebIdentityToken(ctx, token, "")
+	return s.validateWebIdentityToken(ctx, token, "", false)
 }
 
 // ValidateWebIdentityTokenForAccount mirrors ValidateWebIdentityToken but
 // scopes the IAM-managed provider lookup to roleAccountID. Pass "" for
 // callers that don't yet know the account (e.g. claim-based mode).
 func (s *STSService) ValidateWebIdentityTokenForAccount(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
-	return s.validateWebIdentityToken(ctx, token, roleAccountID)
+	return s.validateWebIdentityToken(ctx, token, roleAccountID, false)
+}
+
+// authenticateOIDCProvider authenticates the token, consuming it for replay
+// protection when the caller is the single-use STS exchange and the provider
+// supports it. Other providers and non-exchange callers authenticate without
+// consuming.
+func authenticateOIDCProvider(ctx context.Context, provider providers.IdentityProvider, token string, singleUse bool) (*providers.ExternalIdentity, error) {
+	if singleUse {
+		if su, ok := provider.(interface {
+			AuthenticateSingleUse(context.Context, string) (*providers.ExternalIdentity, error)
+		}); ok {
+			return su.AuthenticateSingleUse(ctx, token)
+		}
+	}
+	return provider.Authenticate(ctx, token)
 }
 
 // extractIssuerFromJWT extracts the issuer (iss) claim from a JWT token without verification

weed/iamapi/iamapi_server.go

@@ -42,10 +42,11 @@ type IamS3ApiConfigure struct {
 }
 
 type IamServerOption struct {
-	Masters        map[string]pb.ServerAddress
-	Filers         []pb.ServerAddress
-	Port           int
-	GrpcDialOption grpc.DialOption
+	Masters                     map[string]pb.ServerAddress
+	Filers                      []pb.ServerAddress
+	Port                        int
+	GrpcDialOption              grpc.DialOption
+	IamSelfServiceRequiresGrant bool
 }
 
 type IamApiServer struct {
@@ -87,8 +88,9 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
 	s3ApiConfigure = configure
 
 	s3Option := s3api.S3ApiServerOption{
-		Filers:         option.Filers,
-		GrpcDialOption: option.GrpcDialOption,
+		Filers:                      option.Filers,
+		GrpcDialOption:              option.GrpcDialOption,
+		IamSelfServiceRequiresGrant: option.IamSelfServiceRequiresGrant,
 	}
 
 	// Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified)

weed/mount/peer_registrar_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
 )
 
 // fakeFilerClient captures MountRegister/MountList calls and lets the test
@@ -17,22 +18,21 @@ type fakeFilerClient struct {
 	filer_pb.SeaweedFilerClient // embed for methods we don't need
 
 	mu            sync.Mutex
-	registerCalls []filer_pb.MountRegisterRequest
+	registerCalls []*filer_pb.MountRegisterRequest
 	listResponse  filer_pb.MountListResponse
 }
 
 func (f *fakeFilerClient) MountRegister(ctx context.Context, req *filer_pb.MountRegisterRequest, opts ...grpc.CallOption) (*filer_pb.MountRegisterResponse, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	f.registerCalls = append(f.registerCalls, *req)
+	f.registerCalls = append(f.registerCalls, proto.Clone(req).(*filer_pb.MountRegisterRequest))
 	return &filer_pb.MountRegisterResponse{}, nil
 }
 
 func (f *fakeFilerClient) MountList(ctx context.Context, req *filer_pb.MountListRequest, opts ...grpc.CallOption) (*filer_pb.MountListResponse, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	resp := f.listResponse // value copy
-	return &resp, nil
+	return proto.Clone(&f.listResponse).(*filer_pb.MountListResponse), nil
 }
 
 // fakeFilerFleet maps each addr to its own fakeFilerClient so a test can

weed/operation/assign_file_id.go

@@ -157,11 +157,11 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
 	return ret, lastError
 }
 
-func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
+func LookupJwt(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
 
-	WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+	WithMasterServerClient(ctx, false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
-		resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
+		resp, grpcErr := masterClient.LookupVolume(ctx, &master_pb.LookupVolumeRequest{
 			VolumeOrFileIds: []string{fileId},
 		})
 		if grpcErr != nil {

weed/operation/chunked_file.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
@@ -101,6 +102,9 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
 	if offset > 0 {
 		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
 	}
+	if jwt != "" {
+		req.Header.Set("Authorization", security.BearerPrefix+jwt)
+	}
 
 	resp, err := util_http.Do(req)
 	if err != nil {
@@ -118,7 +122,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
 	case http.StatusPartialContent:
 		break
 	default:
-		return written, fmt.Errorf("Read chunk needle error: [%d] %s", resp.StatusCode, fileUrl)
+		return written, util_http.StatusError(fileUrl, resp)
 
 	}
 	return io.Copy(w, resp.Body)

weed/operation/submit.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 
@@ -50,6 +51,10 @@ type StoragePreference struct {
 
 type GetMasterFn func(ctx context.Context) pb.ServerAddress
 
+// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
+// once; the batch assign above already fetched all their file ids.
+const submitUploadWorkers = 4
+
 func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
 	results := make([]SubmitResult, len(files))
 	var totalBytes int64
@@ -72,8 +77,11 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
 	}
 	ret, err := Assign(context.Background(), masterFn, grpcDialOption, ar)
 	if err != nil {
-		for index := range files {
+		for index, file := range files {
 			results[index].Error = err.Error()
+			if closer, ok := file.Reader.(io.Closer); ok {
+				closer.Close()
+			}
 		}
 		return results, err
 	}
@@ -87,14 +95,35 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
 			file.Server = ret.PublicUrl
 		}
 		file.Pref = pref
-		results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
-		if err != nil {
-			results[index].Error = err.Error()
-		}
 		results[index].Fid = file.Fid
 		results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
 	}
-	return results, nil
+
+	var wg sync.WaitGroup
+	var lastErrLock sync.Mutex
+	jobs := make(chan int)
+	for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for index := range jobs {
+				size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
+				results[index].Size = size
+				if uploadErr != nil {
+					results[index].Error = uploadErr.Error()
+					lastErrLock.Lock()
+					err = uploadErr
+					lastErrLock.Unlock()
+				}
+			}
+		}()
+	}
+	for index := range files {
+		jobs <- index
+	}
+	close(jobs)
+	wg.Wait()
+	return results, err
 }
 
 func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
@@ -118,6 +147,8 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
 	fi, fiErr := fh.Stat()
 	if fiErr != nil {
 		glog.V(0).Info("Failed to stat file:", fullPathFilename)
+		fh.Close()
+		ret.Reader = nil
 		return ret, fiErr
 	}
 	ret.ModTime = fi.ModTime().UTC().Unix()

weed/operation/upload_chunked.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"net/http"
 	"sort"
 	"sync"
 	"time"
@@ -45,6 +46,7 @@ type ChunkedUploadOption struct {
 	Cipher          bool // encrypt data on volume servers
 	AssignFunc      func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error)
 	UploadFunc      func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) // Optional: for testing
+	LookupJwtFunc   func(ctx context.Context, fileId string) security.EncodedJwt                        // Optional: renews the volume JWT for a cleanup delete
 }
 
 var chunkBufferPool = sync.Pool{
@@ -213,7 +215,7 @@ uploadLoop:
 				// failed write can still leave a copy behind. No chunk will name
 				// this fid whether we retry or give up here, and an unreferenced
 				// needle is not garbage vacuum can find, so drop it either way.
-				deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt)
+				deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt, opt)
 				if attempt == chunkAssignAttempts || !ShouldReassignUpload(uploadResultErr) || objectFailed() {
 					break
 				}
@@ -413,19 +415,46 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data
 	if firstErr != nil {
 		// A failed fan-out records no chunk, so roll back the copies that landed
 		// before the cancel rather than leaking them as orphans.
-		deleteChunkFromHolders(succeeded, fid, jwt)
+		deleteChunkFromHolders(succeeded, fid, jwt, opt)
 		return nil, firstErr
 	}
 	return first, nil
 }
 
+// cleanupJwtLookupTimeout bounds a cleanup-token renewal, covering the wait
+// for a master plus the lookup RPC, so a master outage cannot stall the
+// rollback of a failed fan-out.
+const cleanupJwtLookupTimeout = 10 * time.Second
+
 // deleteChunkFromHolders best-effort removes a needle from each holder it landed
 // on, using type=replicate so the volume drops only its local copy. A failed
 // delete falls back to vacuum reclaiming the orphan.
-func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt) {
+func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt, opt *ChunkedUploadOption) {
+	// An unresponsive holder can stall the upload past the write JWT's expiry,
+	// so renew the token whenever a holder rejects the delete as unauthorized.
+	// Each holder's delete is retried at most once, and a slow holder that
+	// outlives the renewed token can still trigger another renewal.
+	deleteJwt := jwt
 	for _, host := range hosts {
-		if err := util_http.Delete(fmt.Sprintf("http://%s/%s?type=replicate", host, fid), string(jwt)); err != nil {
+		deleteUrl := fmt.Sprintf("http://%s/%s?type=replicate", host, fid)
+		body, status, err := util_http.DeleteProxied(deleteUrl, string(deleteJwt))
+		if status == http.StatusUnauthorized && opt.LookupJwtFunc != nil {
+			lookupCtx, cancel := context.WithTimeout(context.Background(), cleanupJwtLookupTimeout)
+			token := opt.LookupJwtFunc(lookupCtx, fid)
+			cancel()
+			if token != "" {
+				deleteJwt = token
+				body, status, err = util_http.DeleteProxied(deleteUrl, string(deleteJwt))
+			}
+		}
+		if err != nil {
 			glog.Warningf("replica fan-out cleanup: delete %s from %s: %v", fid, host, err)
+			continue
+		}
+		switch status {
+		case http.StatusNotFound, http.StatusNoContent, http.StatusAccepted, http.StatusOK:
+		default:
+			glog.Warningf("replica fan-out cleanup: delete %s from %s: status %d: %s", fid, host, status, body)
 		}
 	}
 }

weed/operation/upload_chunked_test.go

@@ -26,13 +26,11 @@ func TestUploadReaderInChunksReturnsPartialResultsOnError(t *testing.T) {
 	testData := bytes.Repeat([]byte("test data for chunk upload failure testing"), 1000) // ~40KB
 	reader := bytes.NewReader(testData)
 
-	uploadAttempts := 0
+	var uploadAttempts int32
 
 	// Create a mock assign function that succeeds for first chunk, then fails
 	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
-		uploadAttempts++
-
-		if uploadAttempts == 1 {
+		if atomic.AddInt32(&uploadAttempts, 1) == 1 {
 			// First chunk succeeds
 			return nil, &AssignResult{
 				Fid:       "test-fid-1,1234",
@@ -679,3 +677,90 @@ func TestUploadReaderInChunksAppendsFsyncWhenAssigned(t *testing.T) {
 		t.Errorf("Expected the upload URL to carry ?fsync=true when the assignment says fsync, got %q", gotUploadUrl)
 	}
 }
+
+// A holder that stalls past the assignment JWT's expiry answers the rollback
+// delete with 401; the cleanup renews the token instead of leaking the copy.
+func TestDeleteChunkFromHoldersRenewsExpiredJwt(t *testing.T) {
+	var staleDeletes, freshDeletes int32
+	holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			w.WriteHeader(http.StatusCreated)
+			return
+		}
+		switch r.Header.Get("Authorization") {
+		case "Bearer fresh-jwt":
+			atomic.AddInt32(&freshDeletes, 1)
+			w.WriteHeader(http.StatusAccepted)
+		default:
+			atomic.AddInt32(&staleDeletes, 1)
+			w.WriteHeader(http.StatusUnauthorized)
+			fmt.Fprint(w, `{"error":"wrong jwt"}`)
+		}
+	}))
+	defer holder.Close()
+
+	lookups := 0
+	deleteChunkFromHolders(
+		[]string{strings.TrimPrefix(holder.URL, "http://")},
+		"1,0a0b0c0d",
+		"stale-jwt",
+		&ChunkedUploadOption{
+			LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt {
+				lookups++
+				return "fresh-jwt"
+			},
+		},
+	)
+
+	if lookups != 1 {
+		t.Fatalf("expected one jwt renewal, got %d", lookups)
+	}
+	if atomic.LoadInt32(&freshDeletes) != 1 {
+		t.Fatalf("expected the delete retried with the fresh jwt, got %d fresh deletes", freshDeletes)
+	}
+}
+
+// A later holder that rejects the renewed token can trigger another renewal:
+// the one-retry limit is per holder, not per cleanup pass.
+func TestDeleteChunkFromHoldersRenewsPerHolder(t *testing.T) {
+	var acceptedDeletes int32
+	holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			w.WriteHeader(http.StatusCreated)
+			return
+		}
+		if r.Header.Get("Authorization") == "Bearer jwt-2" {
+			atomic.AddInt32(&acceptedDeletes, 1)
+			w.WriteHeader(http.StatusAccepted)
+			return
+		}
+		w.WriteHeader(http.StatusUnauthorized)
+		fmt.Fprint(w, `{"error":"wrong jwt"}`)
+	}))
+	defer holder.Close()
+
+	tokens := []security.EncodedJwt{"jwt-1", "jwt-2"}
+	lookups := 0
+	deleteChunkFromHolders(
+		[]string{
+			strings.TrimPrefix(holder.URL, "http://"),
+			strings.TrimPrefix(holder.URL, "http://"),
+		},
+		"1,0a0b0c0d",
+		"stale-jwt",
+		&ChunkedUploadOption{
+			LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt {
+				token := tokens[lookups]
+				lookups++
+				return token
+			},
+		},
+	)
+
+	if lookups != 2 {
+		t.Fatalf("expected a renewal per rejecting holder, got %d", lookups)
+	}
+	if atomic.LoadInt32(&acceptedDeletes) != 1 {
+		t.Fatalf("expected the second holder's retried delete to land, got %d", acceptedDeletes)
+	}
+}

weed/pb/filer_pb/filer_client.go

@@ -67,6 +67,16 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.Fu
 // consumers would mistake for an entry.
 const ListSnapshotTsNsTrailerKey = "sw-list-snapshot-ts-ns"
 
+// TraverseModeHeaderKey is gRPC metadata on a TraverseBfsMetadata stream. As
+// request metadata, TraverseModeSnapshot asks for the entries from one store
+// snapshot (any order, entry_ts_ns left 0) instead of a breadth-first walk; as
+// a response header it says what the filer did. An older filer sends none.
+const (
+	TraverseModeHeaderKey = "sw-traverse-mode"
+	TraverseModeSnapshot  = "snapshot"
+	TraverseModeBfs       = "bfs"
+)
+
 type EachEntryFunction func(entry *Entry, isLast bool) error
 
 func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (err error) {

weed/plugin/worker/worker.go

@@ -660,8 +660,7 @@ func (w *Worker) buildHeartbeat() *plugin_pb.WorkerHeartbeat {
 		if work == nil {
 			continue
 		}
-		cloned := *work
-		running = append(running, &cloned)
+		running = append(running, proto.Clone(work).(*plugin_pb.RunningWork))
 	}
 	w.runningMu.RUnlock()
 

weed/remote_storage/s3/s3_storage_client.go

@@ -168,6 +168,27 @@ func S3CompatibleEndpoint(conf *remote_pb.RemoteConf) (string, bool) {
 	return opt.endpoint, true
 }
 
+// awsCredentialSourceConfigured reports whether the environment names a source
+// for the AWS SDK default credential chain, such as static keys, a profile,
+// IRSA web identity or the ECS/EKS container credentials endpoint.
+func awsCredentialSourceConfigured() bool {
+	for _, name := range []string{
+		"AWS_ACCESS_KEY_ID",
+		"AWS_PROFILE",
+		"AWS_DEFAULT_PROFILE",
+		"AWS_WEB_IDENTITY_TOKEN_FILE",
+		"AWS_CONTAINER_CREDENTIALS_FULL_URI",
+		"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
+		"AWS_SHARED_CREDENTIALS_FILE",
+		"AWS_CONFIG_FILE",
+	} {
+		if os.Getenv(name) != "" {
+			return true
+		}
+	}
+	return false
+}
+
 // MakeWithHTTPClient builds the client for any S3-SDK-backed remote storage
 // type using the supplied *http.Client (or the AWS SDK default when nil).
 // Callers that need to pin the dial path against DNS rebinding can pass a
@@ -191,12 +212,32 @@ func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (re
 	}
 	if opt.accessKey != "" && opt.secretKey != "" {
 		config.Credentials = credentials.NewStaticCredentials(opt.accessKey, opt.secretKey, "")
-	} else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" {
-		// Explicitly disable signing for public buckets.
+	} else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" && !awsCredentialSourceConfigured() {
+		// Explicitly disable signing for public buckets. When the environment
+		// names an AWS credential source, leave Credentials nil so the SDK
+		// default chain (env, web identity, container, shared profile) is used.
 		config.Credentials = credentials.AnonymousCredentials
+	} else if config.Credentials == nil && config.HTTPClient != nil {
+		// Remote credential providers (container endpoints, IMDS, web
+		// identity) fetch through this HTTP client; the guarded one is for the
+		// S3 endpoint and blocks the link-local credential endpoints, so the
+		// default chain resolves on an unguarded client. Shared config stays
+		// enabled so web identity and profiles resolve here too, and the
+		// region carries over so STS can sign assume-role requests.
+		credSess, err := session.NewSessionWithOptions(session.Options{
+			Config:            aws.Config{Region: config.Region, HTTPClient: &http.Client{}},
+			SharedConfigState: session.SharedConfigEnable,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("create %s credential session: %w", opt.name, err)
+		}
+		config.Credentials = credSess.Config.Credentials
 	}
 
-	sess, err := session.NewSession(config)
+	sess, err := session.NewSessionWithOptions(session.Options{
+		Config:            *config,
+		SharedConfigState: session.SharedConfigEnable,
+	})
 	if err != nil {
 		return nil, fmt.Errorf("create %s session: %w", opt.name, err)
 	}

weed/remote_storage/s3/s3_storage_client_test.go

@@ -21,6 +21,7 @@ import (
 )
 
 func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) {
+	clearAWSCredentialEnv(t)
 	maker := s3RemoteStorageMaker{}
 	conf := &remote_pb.RemoteConf{
 		Type:             "s3",
@@ -40,6 +41,74 @@ func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) {
 	require.Same(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
 }
 
+func clearAWSCredentialEnv(t *testing.T) {
+	t.Helper()
+	for _, name := range []string{
+		"AWS_ACCESS_KEY_ID",
+		"AWS_PROFILE",
+		"AWS_DEFAULT_PROFILE",
+		"AWS_WEB_IDENTITY_TOKEN_FILE",
+		"AWS_CONTAINER_CREDENTIALS_FULL_URI",
+		"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
+		"AWS_SHARED_CREDENTIALS_FILE",
+		"AWS_CONFIG_FILE",
+	} {
+		t.Setenv(name, "")
+	}
+}
+
+func TestS3MakeUsesDefaultCredentialChainWhenKeysAreEmptyAndSourceIsConfigured(t *testing.T) {
+	for name, value := range map[string]string{
+		"AWS_ACCESS_KEY_ID":                      "env-access",
+		"AWS_PROFILE":                            "some-profile",
+		"AWS_WEB_IDENTITY_TOKEN_FILE":            "/var/run/secrets/token",
+		"AWS_CONTAINER_CREDENTIALS_FULL_URI":     "http://127.0.0.1:1/creds",
+		"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/creds",
+	} {
+		t.Run(name, func(t *testing.T) {
+			clearAWSCredentialEnv(t)
+			t.Setenv(name, value)
+			t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/test")
+
+			remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{
+				Type:             "s3",
+				S3Region:         "us-east-1",
+				S3Endpoint:       "http://localhost:8333",
+				S3ForcePathStyle: true,
+			})
+			require.NoError(t, err)
+
+			s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3)
+			require.True(t, ok)
+			require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+		})
+	}
+}
+
+// A caller that pins a custom http.Client still gets the default chain, so
+// remote providers resolve on an unguarded client while S3 requests use the
+// pinned one.
+func TestS3MakeResolvesEnvCredentialsWithCustomHTTPClient(t *testing.T) {
+	clearAWSCredentialEnv(t)
+	t.Setenv("AWS_ACCESS_KEY_ID", "env-access")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret")
+
+	remoteClient, err := MakeWithHTTPClient(&remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+	}, &http.Client{})
+	require.NoError(t, err)
+
+	s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3)
+	require.True(t, ok)
+	require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+	value, err := s3Client.Config.Credentials.Get()
+	require.NoError(t, err)
+	require.Equal(t, "env-access", value.AccessKeyID)
+}
+
 func TestS3MakeUsesStaticCredentialsWhenKeysAreProvided(t *testing.T) {
 	maker := s3RemoteStorageMaker{}
 	conf := &remote_pb.RemoteConf{
@@ -396,3 +465,24 @@ func TestIsNoSuchKey(t *testing.T) {
 	require.True(t, isNoSuchKey(fmt.Errorf("download: %w", awserr.New(awss3.ErrCodeNoSuchKey, "missing", nil))))
 	require.False(t, isNoSuchKey(awserr.New(awss3.ErrCodeNoSuchBucket, "missing", nil)))
 }
+
+func TestS3MakeResolvesEnvCredentialsWhenAnonymousDefault(t *testing.T) {
+	clearAWSCredentialEnv(t)
+	t.Setenv("AWS_ACCESS_KEY_ID", "env-access")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret")
+
+	remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+	})
+	require.NoError(t, err)
+
+	s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3)
+	require.True(t, ok)
+	value, err := s3Client.Config.Credentials.Get()
+	require.NoError(t, err)
+	require.Equal(t, "env-access", value.AccessKeyID)
+	require.Equal(t, "env-secret", value.SecretAccessKey)
+}

weed/replication/sink/filersink/fetch_write.go

@@ -229,8 +229,7 @@ func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk
 
 	manifestDataChunks := make([]*filer_pb.FileChunk, len(replicatedResolvedChunks))
 	for i, chunk := range replicatedResolvedChunks {
-		copied := *chunk
-		manifestDataChunks[i] = &copied
+		manifestDataChunks[i] = proto.Clone(chunk).(*filer_pb.FileChunk)
 	}
 	filer_pb.BeforeEntrySerialization(manifestDataChunks)
 	manifestData, err := proto.Marshal(&filer_pb.FileChunkManifest{

weed/s3api/auth_access_key_attribution_test.go

@@ -0,0 +1,75 @@
+package s3api
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+const (
+	sharedKeyA    = "AKIASHAREDKEYAAAAAAA"
+	sharedSecretA = "secret-a-wJalrXUtnFEMI/K7MDENG/bPxRfiCY"
+	sharedKeyB    = "AKIASHAREDKEYBBBBBBB"
+	sharedSecretB = "secret-b-wJalrXUtnFEMI/K7MDENG/bPxRfiCY"
+)
+
+// newSharedIdentityIAM returns one identity holding two key pairs, as when
+// several people or services are given their own key on one identity.
+func newSharedIdentityIAM() *IdentityAccessManagement {
+	iam := &IdentityAccessManagement{}
+	shared := &Identity{
+		Name: "shared-user",
+		Credentials: []*Credential{
+			{AccessKey: sharedKeyA, SecretKey: sharedSecretA},
+			{AccessKey: sharedKeyB, SecretKey: sharedSecretB},
+		},
+		Actions: []Action{s3_constants.ACTION_READ},
+	}
+	iam.identities = []*Identity{shared}
+	iam.accessKeyIdent = map[string]*Identity{sharedKeyA: shared, sharedKeyB: shared}
+	iam.nameToIdentity = map[string]*Identity{shared.Name: shared}
+	return iam
+}
+
+// Two keys of one identity: the audit entry names the same requester for both
+// and tells them apart by the key each request was signed with.
+func TestAuditLogCarriesTheSigningAccessKey(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	for _, key := range []struct{ id, secret string }{{sharedKeyA, sharedSecretA}, {sharedKeyB, sharedSecretB}} {
+		req, err := newTestRequest(http.MethodGet, "https://example.com/bucket/ns/t/data/f.parquet", 0, nil)
+		require.NoError(t, err)
+		require.NoError(t, signRequestV4(req, key.id, key.secret))
+		// track() installs the holder before authentication runs.
+		outer := s3_constants.EnsureIdentityHolder(req)
+
+		identity, errCode := iam.reqSignatureV4Verify(outer)
+		require.Equal(t, s3err.ErrNone, errCode)
+		iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrNone, func(http.ResponseWriter, *http.Request) {})
+
+		log := s3err.GetAccessLog(outer, http.StatusOK, s3err.ErrNone)
+		assert.Equal(t, "shared-user", log.Requester)
+		assert.Equal(t, key.id, log.AccessKeyId, "audit entry must name the key the request was signed with")
+	}
+}
+
+// A request refused after authentication still names its key, like its requester.
+func TestAuditLogCarriesTheAccessKeyOnTheDenyPath(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	req, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil)
+	require.NoError(t, err)
+	require.NoError(t, signRequestV4(req, sharedKeyB, sharedSecretB))
+	outer := s3_constants.EnsureIdentityHolder(req)
+	identity, errCode := iam.reqSignatureV4Verify(outer)
+	require.Equal(t, s3err.ErrNone, errCode)
+
+	iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrAccessDenied, func(http.ResponseWriter, *http.Request) {
+		t.Fatal("handler must not run on the deny path")
+	})
+	assert.Equal(t, sharedKeyB, s3err.GetAccessLog(outer, http.StatusForbidden, s3err.ErrAccessDenied).AccessKeyId)
+}

weed/s3api/auth_access_key_post_policy_test.go

@@ -0,0 +1,86 @@
+package s3api
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// signedPostFormV2 builds a multipart POST Object request whose policy is
+// signed with SigV2 under postAuthzAccessKey and secret.
+func signedPostFormV2(t *testing.T, key, secret string) *http.Request {
+	t.Helper()
+	policyJSON := fmt.Sprintf(`{"expiration":"%s","conditions":[["eq","$bucket","%s"],["eq","$key","%s"]]}`,
+		time.Now().UTC().Add(time.Hour).Format("2006-01-02T15:04:05.000Z"), postAuthzBucket, key)
+	encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON))
+
+	var buf bytes.Buffer
+	writer := multipart.NewWriter(&buf)
+	for _, field := range [][2]string{{"bucket", postAuthzBucket}, {"key", key}, {"AWSAccessKeyId", postAuthzAccessKey},
+		{"policy", encodedPolicy}, {"Signature", calculateSignatureV2(encodedPolicy, secret)}} {
+		require.NoError(t, writer.WriteField(field[0], field[1]))
+	}
+	filePart, err := writer.CreateFormFile("file", "payload.txt")
+	require.NoError(t, err)
+	_, err = filePart.Write([]byte("payload"))
+	require.NoError(t, err)
+	require.NoError(t, writer.Close())
+
+	req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, &buf)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	return mux.SetURLVars(req, map[string]string{"bucket": postAuthzBucket})
+}
+
+// A browser POST upload is verified in the handler from its form fields; the
+// key of the verified form reaches the audit log like a signed PUT's. The
+// bucket policy denies denied/ so the request ends before the filer.
+func TestPostPolicyUploadRecordsTheSigningKey(t *testing.T) {
+	s3a := newPostAuthzServer(t)
+	for name, req := range map[string]*http.Request{
+		"SigV4": newSignedPostRequest(t, "denied/secret.txt"),
+		"SigV2": signedPostFormV2(t, "denied/secret.txt", postAuthzSecretKey),
+	} {
+		outer := s3_constants.EnsureIdentityHolder(req)
+		rec := httptest.NewRecorder()
+		s3a.PostPolicyBucketHandler(rec, outer)
+		require.Equal(t, http.StatusForbidden, rec.Code, "%s: %s", name, rec.Body.String())
+		assert.Equal(t, postAuthzAccessKey, s3err.GetAccessLog(outer, http.StatusForbidden, s3err.ErrAccessDenied).AccessKeyId, name)
+	}
+
+	// A form whose signature does not verify records nothing.
+	outer := s3_constants.EnsureIdentityHolder(signedPostFormV2(t, "denied/secret.txt", "not-the-secret"))
+	rec := httptest.NewRecorder()
+	s3a.PostPolicyBucketHandler(rec, outer)
+	require.Contains(t, rec.Body.String(), "SignatureDoesNotMatch")
+	assert.Equal(t, "", s3_constants.GetAccessKeyIdFromContext(outer))
+}
+
+// A signature that verifies for one virtual-hosted bucket but is routed to
+// another is refused after verification; the refusal still names the key.
+func TestSignedVirtualHostRetargetRecordsTheSigningKey(t *testing.T) {
+	iam := newVirtualHostTestIAM("s3.test")
+	r, err := newTestRequest("GET", "http://alpha-bkt.s3.test/shared.txt", 0, nil)
+	require.NoError(t, err)
+	r.Header.Set("Host", "alpha-bkt.s3.test")
+	signV4WithPath(r, "access_key_1", "secret_key_1", r.URL.Path)
+	r.Host = "beta-bkt.s3.test"
+	r.Header.Set("X-Forwarded-Host", "alpha-bkt.s3.test")
+	r = mux.SetURLVars(r, map[string]string{"bucket": "beta-bkt", "object": "shared.txt"})
+	r = s3_constants.EnsureIdentityHolder(r)
+
+	_, _, errCode := iam.doesSignatureMatch(r)
+	require.Equal(t, s3err.ErrAccessDenied, errCode)
+	assert.Equal(t, "access_key_1", s3_constants.GetAccessKeyIdFromContext(r))
+}

weed/s3api/auth_access_key_shapes_test.go

@@ -0,0 +1,141 @@
+package s3api
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// The key recorded for a request is the key of the credential that verified,
+// whatever else the request carries. A request names its key in several
+// places (SigV4 header, X-Amz-Credential, SigV2 header, AWSAccessKeyId);
+// getRequestAuthType picks one and that verifier records it, so a second,
+// unverified key in another place never reaches the audit log or a vended
+// session.
+
+func authAndRecordedKey(t *testing.T, iam *IdentityAccessManagement, r *http.Request) (s3err.ErrorCode, string) {
+	t.Helper()
+	r = s3_constants.EnsureIdentityHolder(r)
+	_, errCode, _ := iam.authenticateRequestInternal(r)
+	return errCode, s3_constants.GetAccessKeyIdFromContext(r)
+}
+
+func credOf(key, secret string) *Credential { return &Credential{AccessKey: key, SecretKey: secret} }
+
+func TestRecordedKeyIsTheVerifiedKeyForSigV4Header(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	r, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil)
+	require.NoError(t, err)
+	require.NoError(t, signRequestV4(r, sharedKeyA, sharedSecretA))
+	errCode, got := authAndRecordedKey(t, iam, r)
+	require.Equal(t, s3err.ErrNone, errCode)
+	assert.Equal(t, sharedKeyA, got)
+}
+
+// SigV2 header signed with A, plus X-Amz-Credential naming B or garbage: the
+// request authenticates as A (SigV2 is checked first) and A is recorded.
+func TestRecordedKeyIgnoresAJunkCredentialQueryOnASigV2Request(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	for _, junk := range []string{sharedKeyB + "/20261016/us-east-1/s3/aws4_request", "GARBAGE"} {
+		r, err := http.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet?X-Amz-Credential="+url.QueryEscape(junk), nil)
+		require.NoError(t, err)
+		r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
+		r.Header.Set("Authorization", signatureV2(credOf(sharedKeyA, sharedSecretA), r.Method, r.URL.EscapedPath(), r.URL.Query().Encode(), r.Header))
+		errCode, got := authAndRecordedKey(t, iam, r)
+		require.Equal(t, s3err.ErrNone, errCode, "the SigV2 signature with A verifies (junk %q)", junk)
+		assert.Equal(t, sharedKeyA, got, "the key that verified, not the junk %q", junk)
+	}
+}
+
+// SigV2 presigned with A, plus a junk SigV4 Authorization naming B: the
+// presigned A verifies and A is recorded.
+func TestRecordedKeyIgnoresAJunkSigV4HeaderOnASigV2PresignedRequest(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	expires := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10)
+	q := url.Values{"AWSAccessKeyId": {sharedKeyA}, "Expires": {expires}}
+	r, err := http.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet?"+q.Encode(), nil)
+	require.NoError(t, err)
+	sig := preSignatureV2(credOf(sharedKeyA, sharedSecretA), r.Method, r.URL.EscapedPath(), r.URL.Query().Encode(), r.Header, expires)
+	q.Set("Signature", sig)
+	r.URL.RawQuery = q.Encode()
+	r.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+sharedKeyB+"/20261016/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=00")
+
+	errCode, got := authAndRecordedKey(t, iam, r)
+	require.Equal(t, s3err.ErrNone, errCode, "the SigV2 presigned A verifies (SigV2 is dispatched first)")
+	assert.Equal(t, sharedKeyA, got, "the presigned key A is recorded, never the unverified B")
+}
+
+// A refused signature records nothing, even when the key exists.
+func TestRecordedKeyIsEmptyWhenTheSignatureFails(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	r, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil)
+	require.NoError(t, err)
+	require.NoError(t, signRequestV4(r, sharedKeyA, "not-the-secret"))
+	errCode, got := authAndRecordedKey(t, iam, r)
+	require.NotEqual(t, s3err.ErrNone, errCode)
+	assert.Equal(t, "", got)
+
+	r2 := httptest.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet", nil)
+	r2.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
+	r2.Header.Set("Authorization", "AWS "+sharedKeyB+":bm90LWEtc2lnbmF0dXJl")
+	errCode, got = authAndRecordedKey(t, iam, r2)
+	require.NotEqual(t, s3err.ErrNone, errCode)
+	assert.Equal(t, "", got)
+}
+
+// A streaming (aws-chunked) upload records the key of its seed signature.
+func TestRecordedKeyForAStreamingUpload(t *testing.T) {
+	iam := newSharedIdentityIAM()
+	now := time.Now().UTC()
+	amzDate, dateStamp := now.Format(iso8601Format), now.Format(yyyymmdd)
+	scope := dateStamp + "/us-east-1/s3/aws4_request"
+	hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+	canonicalHeaders := "content-encoding:aws-chunked\n" + "host:s3.amazonaws.com\n" +
+		"x-amz-content-sha256:" + hashedPayload + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-decoded-content-length:4\n"
+	signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length"
+	canonicalRequest := "PUT\n/bucket/ns/t/data/f.parquet\n\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + hashedPayload
+	stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + getSHA256Hash([]byte(canonicalRequest))
+	seed := getSignature(getSigningKey(sharedSecretB, dateStamp, "us-east-1", "s3"), stringToSign)
+
+	r, err := http.NewRequest(http.MethodPut, "http://s3.amazonaws.com/bucket/ns/t/data/f.parquet", bytes.NewReader([]byte("ignored")))
+	require.NoError(t, err)
+	r.Header.Set("Host", "s3.amazonaws.com")
+	r.Header.Set("x-amz-date", amzDate)
+	r.Header.Set("x-amz-content-sha256", hashedPayload)
+	r.Header.Set("Content-Encoding", "aws-chunked")
+	r.Header.Set("x-amz-decoded-content-length", "4")
+	r.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", sharedKeyB, scope, signedHeaders, seed))
+
+	errCode, got := authAndRecordedKey(t, iam, r)
+	require.Equal(t, s3err.ErrNone, errCode)
+	assert.Equal(t, sharedKeyB, got)
+}
+
+// An STS session's request records the session's temporary key.
+func TestRecordedKeyForAnSTSSession(t *testing.T) {
+	stsService, config := setupTestSTSService(t)
+	iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
+	iam.SetIAMIntegration(&S3IAMIntegration{stsService: stsService, enabled: true})
+	sessionInfo, sessionToken := issueTestSession(t, stsService, config)
+
+	r, err := newTestRequest(http.MethodGet, "https://example.com/reports/data.csv", 0, nil)
+	require.NoError(t, err)
+	r.Header.Set("X-Amz-Security-Token", sessionToken)
+	require.NoError(t, signRequestV4(r, sessionInfo.Credentials.AccessKeyId, sessionInfo.Credentials.SecretAccessKey))
+	outer := s3_constants.EnsureIdentityHolder(r)
+	identity, errCode := iam.reqSignatureV4Verify(outer)
+	require.Equal(t, s3err.ErrNone, errCode)
+	iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrNone, func(http.ResponseWriter, *http.Request) {})
+	assert.Equal(t, sessionInfo.Credentials.AccessKeyId, s3err.GetAccessLog(outer, http.StatusOK, s3err.ErrNone).AccessKeyId)
+}

weed/s3api/auth_copy_source_test.go

@@ -3,6 +3,7 @@ package s3api
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -251,9 +252,10 @@ func TestAuthorizeCopySource_PresignedURLSessionTokenPreserved(t *testing.T) {
 	}
 
 	// Presigned URL: session token is in the query string, no
-	// X-Amz-Security-Token header. The destination URL also carries unrelated
-	// SigV4 query params (X-Amz-Algorithm, etc.) that must not leak into the
-	// synthetic source action resolution.
+	// X-Amz-Security-Token header. The destination URL also carries the rest of
+	// the SigV4 presign params, which ride along so auth-derived policy
+	// conditions (s3:authType) match the real request; none of them steer
+	// ResolveS3Action.
 	req, err := http.NewRequest(http.MethodPut,
 		"http://s3.local/bucket-a/dst.bin?X-Amz-Algorithm=AWS4-HMAC-SHA256"+
 			"&X-Amz-Credential=stsKey%2F20260518%2Fus-east-1%2Fs3%2Faws4_request"+
@@ -267,8 +269,12 @@ func TestAuthorizeCopySource_PresignedURLSessionTokenPreserved(t *testing.T) {
 	assert.Equal(t, s3err.ErrNone, errCode)
 	assert.Equal(t, "presigned-session-token", capturedSessionToken,
 		"session token from presigned-URL query must reach the IAM check")
-	// versionId omitted, only the token should be present in the synthetic query.
-	assert.Equal(t, "X-Amz-Security-Token=presigned-session-token", capturedRawQuery)
+	capturedQuery, err := url.ParseQuery(capturedRawQuery)
+	require.NoError(t, err)
+	assert.Equal(t, "presigned-session-token", capturedQuery.Get("X-Amz-Security-Token"))
+	assert.Equal(t, "AWS4-HMAC-SHA256", capturedQuery.Get("X-Amz-Algorithm"))
+	assert.False(t, capturedQuery.Has("versionId"))
+	assert.False(t, capturedQuery.Has("acl"))
 }
 
 // TestAuthorizeCopySource_PreservesCopySourceHeader verifies that the synthetic

weed/s3api/auth_credentials.go

@@ -69,6 +69,11 @@ type IdentityAccessManagement struct {
 	// IAM Integration for advanced features
 	iamIntegration IAMIntegration
 
+	// selfServiceRequiresGrant makes the self-service IAM actions (an
+	// identity's own access keys, GetUser on itself) need an iam:<Action>
+	// grant like every other IAM action. Off by default.
+	selfServiceRequiresGrant bool
+
 	// Serializes resyncs of the policy set into the advanced IAM manager so that
 	// concurrent policy updates can't apply stale views out of order. See
 	// resyncIAMManagerPolicies.
@@ -337,6 +342,7 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient
 		hashCounters: make(map[string]*int32),
 		filerClient:  filerClient,
 	}
+	iam.selfServiceRequiresGrant = option.IamSelfServiceRequiresGrant
 
 	// Always initialize credential manager with fallback to defaults
 	credentialManager, err := credential.NewCredentialManagerWithDefaults(credential.CredentialStoreTypeName(explicitStore))
@@ -3061,11 +3067,21 @@ func (iam *IdentityAccessManagement) bucketsNamedByAttachedPolicies(r *http.Requ
 // its query describe. versionId, when set, and the STS session token ride along
 // for policy conditions. Returns ErrNone when auth is disabled (checked by the
 // callers) or the identity is an admin.
+// authQueryParams are the query keys carrying a request's authentication
+// material (SigV2 and SigV4 presign parameters). None steer ResolveS3Action,
+// so they can ride along on a synthesized request.
+var authQueryParams = []string{
+	"AWSAccessKeyId", "Signature", "Expires",
+	"X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", "X-Amz-Expires",
+	"X-Amz-Signature", "X-Amz-SignedHeaders", "X-Amz-Security-Token",
+}
+
 // objectKeyPolicyRequest synthesizes a <method> /<bucket>/<objectKey> request
 // for policy evaluation against a key the real URL does not name. The query is
 // built from scratch so a param on the real request (delete, uploadId,
 // partNumber, ...) cannot steer ResolveS3Action off the intended action; the
-// STS token and versionId ride along for policy conditions.
+// versionId and the auth-material params ride along so auth-derived policy
+// conditions (s3:authType) evaluate as they did on the real request.
 func objectKeyPolicyRequest(r *http.Request, method, bucket, objectKey, versionId string) *http.Request {
 	keyReq := new(http.Request)
 	*keyReq = *r
@@ -3078,9 +3094,9 @@ func objectKeyPolicyRequest(r *http.Request, method, bucket, objectKey, versionI
 	if versionId != "" {
 		keyQuery.Set("versionId", versionId)
 	}
-	if strings.Contains(r.URL.RawQuery, "X-Amz-Security-Token") {
-		if token := r.URL.Query().Get("X-Amz-Security-Token"); token != "" {
-			keyQuery.Set("X-Amz-Security-Token", token)
+	for _, key := range authQueryParams {
+		if vals, ok := r.URL.Query()[key]; ok {
+			keyQuery[key] = vals
 		}
 	}
 	if len(keyQuery) > 0 {

weed/s3api/auth_signature_v2.go

@@ -18,6 +18,7 @@
 package s3api
 
 import (
+	"context"
 	"crypto/hmac"
 	"crypto/sha1"
 	"crypto/subtle"
@@ -29,6 +30,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
@@ -67,7 +69,7 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide
 	return iam.doesPresignV2SignatureMatch(r)
 }
 
-func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) (*Identity, s3err.ErrorCode) {
+func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) {
 
 	accessKey := formValues.Get("AWSAccessKeyId")
 	if accessKey == "" {
@@ -107,6 +109,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.
 	if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
+	s3_constants.RecordAccessKeyId(ctx, cred.AccessKey)
 	return identity, s3err.ErrNone
 }
 
@@ -164,6 +167,7 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity
 	if !compareSignatureV2(v2Signature, expectedV2Signature) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
+	s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey)
 	return identity, s3err.ErrNone
 }
 
@@ -222,6 +226,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request
 	if !compareSignatureV2(signature, expectedSignature) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
+	s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey)
 	return identity, s3err.ErrNone
 }
 

weed/s3api/auth_signature_v4.go

@@ -19,6 +19,7 @@ package s3api
 
 import (
 	"bytes"
+	"context"
 	"crypto/hmac"
 	"crypto/sha256"
 	"crypto/subtle"
@@ -407,6 +408,10 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
 	}
 
 	if matchedHost != "" {
+		// The key of the credential that verified: the audit log and the
+		// catalog's vended sessions name it (never a key re-read from the request).
+		// Recorded before the retarget check so that refusal names it too.
+		s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey)
 		if signedBucket, ok := bucketFromVirtualHost(matchedHost, iam.domain); ok {
 			if routedBucket, _ := s3_constants.GetBucketAndObject(r); routedBucket != "" && routedBucket != signedBucket {
 				glog.V(2).Infof("reject %s %s: signed host %q implies bucket %q but routed to %q",
@@ -830,7 +835,7 @@ func parseSignedHeaderList(signedHeadersValue string) ([]string, s3err.ErrorCode
 	return signedHeaders, s3err.ErrNone
 }
 
-func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) (*Identity, s3err.ErrorCode) {
+func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) {
 
 	// Parse credential tag.
 	credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential"))
@@ -868,6 +873,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
 	if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
 		return nil, s3err.ErrSignatureDoesNotMatch
 	}
+	s3_constants.RecordAccessKeyId(ctx, cred.AccessKey)
 	return identity, s3err.ErrNone
 }
 

weed/s3api/chunked_reader_v4_test.go

@@ -111,7 +111,7 @@ func TestNewSignV4ChunkedReaderStreamingUnsignedPayloadTrailer(t *testing.T) {
 	runWithRequest(iam, req, t, expectedPayload)
 }
 
-func runWithRequest(iam IdentityAccessManagement, req *http.Request, t *testing.T, expectedPayload string) {
+func runWithRequest(iam *IdentityAccessManagement, req *http.Request, t *testing.T, expectedPayload string) {
 	reader, errCode := iam.newChunkedReader(req)
 	assert.NotNil(t, reader)
 	assert.Equal(t, s3err.ErrNone, errCode)
@@ -124,11 +124,11 @@ func runWithRequest(iam IdentityAccessManagement, req *http.Request, t *testing.
 	assert.Equal(t, expectedPayload, string(data))
 }
 
-func setupIam() IdentityAccessManagement {
+func setupIam() *IdentityAccessManagement {
 	// Create an IdentityAccessManagement instance
 	// Add default access keys and secrets
 
-	iam := IdentityAccessManagement{
+	iam := &IdentityAccessManagement{
 		identities:        []*Identity{},
 		accessKeyIdent:    map[string]*Identity{},
 		accounts:          map[string]*Account{},

weed/s3api/filer_multipart.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"cmp"
+	"container/heap"
 	"context"
 	"crypto/md5"
 	"crypto/rand"
@@ -1292,7 +1293,15 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 
 	glog.V(2).Infof("listMultipartUploads input %v", input)
 
-	entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", "", false, math.MaxInt32)
+	uploadsFolder := s3a.genUploadsFolder(*input.Bucket)
+	var page *multipartUploadPage
+	err := listRetry(uploadsFolder, func() error {
+		page = newMultipartUploadPage(input)
+		return filer_pb.List(context.Background(), s3a, uploadsFolder, "", func(entry *filer_pb.Entry, isLastEntry bool) error {
+			page.add(entry)
+			return nil
+		}, "", false, math.MaxInt32)
+	})
 	if err != nil {
 		// A missing .uploads folder normally lists as empty with no error; a
 		// store that reports it as not-found still means an empty list.
@@ -1304,55 +1313,104 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 		return nil, s3err.ErrInternalError
 	}
 
-	return selectMultipartUploads(input, entries), s3err.ErrNone
+	return page.result(), s3err.ErrNone
 }
 
 // selectMultipartUploads orders uploads by key, then by upload id, and resumes
 // after the (key-marker, upload-id-marker) position, so an aborted marker
 // upload does not hide the rest of its key.
 func selectMultipartUploads(input *s3.ListMultipartUploadsInput, entries []*filer_pb.Entry) *ListMultipartUploadsResult {
+	page := newMultipartUploadPage(input)
+	for _, entry := range entries {
+		page.add(entry)
+	}
+	return page.result()
+}
+
+// multipartUploadPage accumulates one page of a ListMultipartUploads result.
+// Upload order is the object key kept in extended attributes, not the upload
+// id the filer sorts by, so every entry has to be seen; keeping only the
+// maxUploads smallest candidates bounds the heap by the page, not the folder.
+type multipartUploadPage struct {
+	input *s3.ListMultipartUploadsInput
+	limit int
+	top   candidateHeap
+	more  bool
+}
+
+type multipartCandidate struct {
+	key   string
+	entry *filer_pb.Entry
+}
+
+func compareMultipartCandidate(a, b multipartCandidate) int {
+	return cmp.Or(strings.Compare(a.key, b.key), strings.Compare(a.entry.Name, b.entry.Name))
+}
+
+// candidateHeap is a max-heap on (key, upload id): the root sorts last of the
+// candidates held, so popping it keeps the page's smallest entries.
+type candidateHeap []multipartCandidate
+
+func (h candidateHeap) Len() int { return len(h) }
+func (h candidateHeap) Less(i, j int) bool {
+	return compareMultipartCandidate(h[i], h[j]) > 0
+}
+func (h candidateHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+func (h *candidateHeap) Push(x any)   { *h = append(*h, x.(multipartCandidate)) }
+func (h *candidateHeap) Pop() any {
+	old := *h
+	n := len(old)
+	*h = old[:n-1]
+	return old[n-1]
+}
+
+func newMultipartUploadPage(input *s3.ListMultipartUploadsInput) *multipartUploadPage {
+	limit := maxUploadsList
+	if *input.MaxUploads < int64(limit) {
+		limit = int(*input.MaxUploads)
+	}
+	return &multipartUploadPage{input: input, limit: limit}
+}
+
+func (p *multipartUploadPage) add(entry *filer_pb.Entry) {
+	if p.limit <= 0 || !isMultipartUploadEntry(entry) {
+		return
+	}
+	input := p.input
+	key := *objectKey(aws.String(string(entry.Extended[s3_constants.ExtMultipartObjectKey])))
+	if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {
+		return
+	}
+	if *input.KeyMarker != "" && (key < *input.KeyMarker || key == *input.KeyMarker && (*input.UploadIdMarker == "" || entry.Name <= *input.UploadIdMarker)) {
+		return
+	}
+	c := multipartCandidate{key: key, entry: entry}
+	if p.top.Len() >= p.limit {
+		p.more = true
+		if compareMultipartCandidate(c, p.top[0]) >= 0 {
+			return
+		}
+		heap.Pop(&p.top)
+	}
+	heap.Push(&p.top, c)
+}
+
+func (p *multipartUploadPage) result() *ListMultipartUploadsResult {
+	input := p.input
 	output := &ListMultipartUploadsResult{
 		Bucket:         input.Bucket,
 		Delimiter:      input.Delimiter,
 		EncodingType:   input.EncodingType,
 		KeyMarker:      input.KeyMarker,
-		MaxUploads:     input.MaxUploads,
+		MaxUploads:     aws.Int64(int64(p.limit)),
 		Prefix:         input.Prefix,
 		UploadIdMarker: input.UploadIdMarker,
 		IsTruncated:    aws.Bool(false),
 	}
-	if *input.MaxUploads <= 0 {
-		return output
-	}
-	type candidate struct {
-		key   string
-		entry *filer_pb.Entry
-	}
-	var candidates []candidate
-	for _, entry := range entries {
-		if !isMultipartUploadEntry(entry) {
-			continue
-		}
-		key := *objectKey(aws.String(string(entry.Extended[s3_constants.ExtMultipartObjectKey])))
-		if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {
-			continue
-		}
-		if *input.KeyMarker != "" && (key < *input.KeyMarker || key == *input.KeyMarker && (*input.UploadIdMarker == "" || entry.Name <= *input.UploadIdMarker)) {
-			continue
-		}
-		candidates = append(candidates, candidate{key: key, entry: entry})
-	}
-	slices.SortFunc(candidates, func(a, b candidate) int {
-		return cmp.Or(strings.Compare(a.key, b.key), strings.Compare(a.entry.Name, b.entry.Name))
-	})
-
+	candidates := make([]multipartCandidate, len(p.top))
+	copy(candidates, p.top)
+	slices.SortFunc(candidates, compareMultipartCandidate)
 	for _, c := range candidates {
-		if int64(len(output.Upload)) >= *input.MaxUploads {
-			last := output.Upload[len(output.Upload)-1]
-			output.IsTruncated = aws.Bool(true)
-			output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId
-			return output
-		}
 		upload := &s3.MultipartUpload{
 			Key:      aws.String(c.key),
 			UploadId: aws.String(c.entry.Name),
@@ -1362,6 +1420,11 @@ func selectMultipartUploads(input *s3.ListMultipartUploadsInput, entries []*file
 		}
 		output.Upload = append(output.Upload, upload)
 	}
+	if p.more {
+		last := output.Upload[len(output.Upload)-1]
+		output.IsTruncated = aws.Bool(true)
+		output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId
+	}
 	return output
 }
 

weed/s3api/filer_multipart_list_test.go

@@ -1,6 +1,9 @@
 package s3api
 
 import (
+	"context"
+	"fmt"
+	"math"
 	"slices"
 	"testing"
 
@@ -8,6 +11,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 // uploadEntry is an upload folder as createMultipartUpload leaves it.
@@ -112,3 +117,84 @@ func TestSelectMultipartUploadsPaginatesEveryUploadOnce(t *testing.T) {
 		t.Fatalf("listed %d of %d uploads", len(seen), len(entries))
 	}
 }
+
+func TestSelectMultipartUploadsHoldsOnlyThePage(t *testing.T) {
+	// a folder far larger than the page must not accumulate in the collector:
+	// peak memory tracks maxUploads, not the open-upload count
+	page := newMultipartUploadPage(listUploadsInput("", "", "", 3))
+	const total = 20000
+	for i := 0; i < total; i++ {
+		page.add(uploadEntry(fmt.Sprintf("%05d", i), fmt.Sprintf("k%05d", total-1-i), int64(i)))
+	}
+	if page.top.Len() != 3 {
+		t.Fatalf("collector holds %d candidates, want the 3-entry page", page.top.Len())
+	}
+	out := page.result()
+	var got []string
+	for _, u := range out.Upload {
+		got = append(got, *u.Key+"/"+*u.UploadId)
+	}
+	want := []string{"k00000/19999", "k00001/19998", "k00002/19997"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+	if !*out.IsTruncated || *out.NextKeyMarker != "k00002" || *out.NextUploadIdMarker != "19997" {
+		t.Fatalf("truncation markers got (%v, %v, %v)", *out.IsTruncated, *out.NextKeyMarker, *out.NextUploadIdMarker)
+	}
+}
+
+func TestSelectMultipartUploadsCapsRequestedPageSize(t *testing.T) {
+	// max-uploads is caller-controlled; the page stays bounded at the server cap
+	page := newMultipartUploadPage(listUploadsInput("", "", "", 10*maxUploadsList))
+	if page.limit != maxUploadsList {
+		t.Fatalf("page limit %d, want capped at %d", page.limit, maxUploadsList)
+	}
+	if got := *page.result().MaxUploads; got != int64(maxUploadsList) {
+		t.Fatalf("MaxUploads reports %d, want the effective cap %d", got, maxUploadsList)
+	}
+}
+
+// A transient failure mid-stream replays through a fresh page, so the uploads
+// the aborted attempt already handed to the collector must not bleed into the
+// result: each upload lists once and the markers track the retried attempt.
+func TestListMultipartUploadsRetryCollectsEachUploadOnce(t *testing.T) {
+	entries := []*filer_pb.Entry{
+		uploadEntry("1", "c", 10),
+		uploadEntry("2", "a", 30),
+		uploadEntry("3", "b", 10),
+	}
+	client := &listRetryClient{
+		entries:   entries,
+		recvErrs:  []error{status.Error(codes.Unavailable, "transport is closing"), nil},
+		recvAfter: []int{2, 0},
+	}
+	accessor := &listRetryAccessor{client: client}
+	input := listUploadsInput("", "", "", 2)
+
+	var page *multipartUploadPage
+	err := listRetry("/buckets/b/.uploads", func() error {
+		page = newMultipartUploadPage(input)
+		return filer_pb.List(context.Background(), accessor, "/buckets/b/.uploads", "", func(entry *filer_pb.Entry, isLastEntry bool) error {
+			page.add(entry)
+			return nil
+		}, "", false, math.MaxInt32)
+	})
+	if err != nil {
+		t.Fatalf("listRetry: %v", err)
+	}
+	if client.attempts != 2 {
+		t.Fatalf("got %d attempts, want 2", client.attempts)
+	}
+	out := page.result()
+	var got []string
+	for _, u := range out.Upload {
+		got = append(got, *u.Key+"/"+*u.UploadId)
+	}
+	want := []string{"a/2", "b/3"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("got %v, want %v (no entry from the aborted attempt may carry over)", got, want)
+	}
+	if !*out.IsTruncated || *out.NextKeyMarker != "b" || *out.NextUploadIdMarker != "3" {
+		t.Fatalf("truncation markers got (%v, %v, %v)", *out.IsTruncated, *out.NextKeyMarker, *out.NextUploadIdMarker)
+	}
+}

weed/s3api/filer_util.go

@@ -93,17 +93,17 @@ func isRetryableListError(err error) bool {
 	return err != nil && !isFilerNotFound(err) && util.IsTransientError(err)
 }
 
-// listWithRetry replays doList while the filer answers with a transient error.
+// listRetry replays doList while the filer answers with a transient error.
 // Both failure points, the ListEntries call itself and the stream.Recv that
 // follows it, surface as a plain error out of filer_pb.List, so a single retry
-// point above it covers both.
-func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_pb.Entry, isLast bool, err error)) (entries []*filer_pb.Entry, isLast bool, err error) {
+// point above it covers both. A replayed attempt may re-deliver entries the
+// last attempt already saw, so a doList that accumulates must reset first.
+func listRetry(parentDirectoryPath string, doList func() error) (err error) {
 
 	backoff := listRetryInitialBackoff
 	for attempt := 1; ; attempt++ {
-		entries, isLast, err = doList()
-		if err == nil || attempt >= listRetryAttempts || !isRetryableListError(err) {
-			return entries, isLast, err
+		if err = doList(); err == nil || attempt >= listRetryAttempts || !isRetryableListError(err) {
+			return err
 		}
 		glog.V(1).Infof("list %s attempt %d/%d hit a transient error, retrying in %v: %v", parentDirectoryPath, attempt, listRetryAttempts, backoff, err)
 		time.Sleep(backoff)
@@ -112,6 +112,16 @@ func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_p
 
 }
 
+func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_pb.Entry, isLast bool, err error)) (entries []*filer_pb.Entry, isLast bool, err error) {
+
+	err = listRetry(parentDirectoryPath, func() error {
+		entries, isLast, err = doList()
+		return err
+	})
+	return entries, isLast, err
+
+}
+
 // A delete is idempotent at the filer, which answers an entry that is already
 // gone with an empty resp.Error, so a reply the transport dropped can be
 // reissued instead of surfaced: as a 500 on the bucket delete, or as a per-key

weed/s3api/iceberg/commit_helpers.go

@@ -252,3 +252,59 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi
 		Metadata:         newMetadata,
 	}, nil
 }
+
+// authorizeCommit asks the S3 Tables layer, before anything is written, whether
+// the caller may update the table. Both commit paths stage the new metadata
+// file in the table's directory before UpdateTable authorizes the pointer
+// flip, so without this gate a caller who may only read the table could still
+// make the server write and delete files under it. UpdateTable decides again
+// when the pointer flips.
+func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespace []string, tableName, identityName string) *icebergRequestError {
+	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s.tablesManager.AuthorizeUpdateTable(ctx, s3tables.NewManagerClient(client), &s3tables.UpdateTableRequest{
+			TableBucketARN: bucketARN,
+			Namespace:      namespace,
+			Name:           tableName,
+		}, identityName)
+	})
+	if err == nil {
+		return nil
+	}
+	var tableErr *s3tables.S3TablesError
+	if errors.As(err, &tableErr) {
+		switch tableErr.Type {
+		case s3tables.ErrCodeAccessDenied:
+			return &icebergRequestError{http.StatusForbidden, "ForbiddenException", fmt.Sprintf("Not authorized to commit to table %s", tableName)}
+		case s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchNamespace:
+			return &icebergRequestError{http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName)}
+		}
+	}
+	glog.V(1).Infof("Iceberg: commit authorization for %s failed: %v", tableName, err)
+	return &icebergRequestError{http.StatusInternalServerError, "InternalServerError", "Failed to authorize commit: " + err.Error()}
+}
+
+// authorizeViewCommit is authorizeCommit for a view, whose update path stages
+// its metadata file before UpdateView authorizes the pointer flip.
+func (s *Server) authorizeViewCommit(ctx context.Context, bucketARN string, namespace []string, viewName, identityName string) *icebergRequestError {
+	err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s.tablesManager.AuthorizeUpdateView(ctx, s3tables.NewManagerClient(client), &s3tables.UpdateViewRequest{
+			TableBucketARN: bucketARN,
+			Namespace:      namespace,
+			Name:           viewName,
+		}, identityName)
+	})
+	if err == nil {
+		return nil
+	}
+	var tableErr *s3tables.S3TablesError
+	if errors.As(err, &tableErr) {
+		switch tableErr.Type {
+		case s3tables.ErrCodeAccessDenied:
+			return &icebergRequestError{http.StatusForbidden, "ForbiddenException", fmt.Sprintf("Not authorized to commit to view %s", viewName)}
+		case s3tables.ErrCodeNoSuchView, s3tables.ErrCodeNoSuchNamespace:
+			return &icebergRequestError{http.StatusNotFound, "NoSuchViewException", fmt.Sprintf("View does not exist: %s", viewName)}
+		}
+	}
+	glog.V(1).Infof("Iceberg: view commit authorization for %s failed: %v", viewName, err)
+	return &icebergRequestError{http.StatusInternalServerError, "InternalServerError", "Failed to authorize commit: " + err.Error()}
+}

weed/s3api/iceberg/handlers_commit.go

@@ -242,6 +242,10 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
 			writeManagerError(w, err)
 			return
 		}
+		if reqErr := s.authorizeCommit(r.Context(), bucketARN, namespace, tableName, identityName); reqErr != nil {
+			writeError(w, reqErr.status, reqErr.errType, reqErr.message)
+			return
+		}
 
 		location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
 		if location == "" {

weed/s3api/iceberg/handlers_oauth.go

@@ -324,20 +324,28 @@ func mintIcebergToken(identityName, accessKey, secret string, ttlSeconds int) (s
 // authenticateBearer validates a Bearer token from the Authorization header.
 // Returns the identity name, identity object, and whether auth succeeded.
 func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) {
+	identityName, identity, _, ok := s.authenticateBearerKey(r)
+	return identityName, identity, ok
+}
+
+// authenticateBearerKey is authenticateBearer that also returns the access key
+// the token was minted from (its verified access_key claim), so the catalog
+// can tell apart the holders of several keys on one identity.
+func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) {
 	auth := r.Header.Get("Authorization")
 	if auth == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 	if !strings.HasPrefix(strings.ToLower(auth), "bearer ") {
-		return "", nil, false
+		return "", nil, "", false
 	}
 	tokenString := strings.TrimSpace(auth[7:])
 	if tokenString == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	if s.credentialValidator == nil {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	// Parse the token without verification first to get the access key,
@@ -347,18 +355,18 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool)
 	_, _, err := parser.ParseUnverified(tokenString, unverified)
 	if err != nil {
 		glog.V(2).Infof("Iceberg OAuth: failed to parse token: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	if unverified.AccessKey == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	// Look up the credential by access key to get the signing key for verification
 	identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey)
 	if err != nil {
 		glog.V(2).Infof("Iceberg OAuth: failed to get credential for access key: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	signingKey := deriveSigningKey(unverified.AccessKey, secretKey)
@@ -371,10 +379,12 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool)
 	})
 	if err != nil || !verified.Valid {
 		glog.V(2).Infof("Iceberg OAuth: token verification failed: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
-	return identityName, identity, true
+	// The token's signing key is derived from this access key, so a verified
+	// token was minted from it.
+	return identityName, identity, unverified.AccessKey, true
 }
 
 // deriveSigningKey derives a signing key from the access key and secret using HMAC-SHA256.

weed/s3api/iceberg/handlers_transaction.go

@@ -140,6 +140,9 @@ func (s *Server) prepareTableCommit(ctx context.Context, bucketName, bucketARN,
 		glog.V(1).Infof("Iceberg: CommitTransaction GetTable error: %v", err)
 		return nil, &icebergRequestError{http.StatusInternalServerError, "InternalServerError", err.Error()}
 	}
+	if reqErr := s.authorizeCommit(ctx, bucketARN, namespace, tableName, identityName); reqErr != nil {
+		return nil, reqErr
+	}
 
 	location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
 	if location == "" {

weed/s3api/iceberg/handlers_view_update.go

@@ -48,6 +48,10 @@ func (s *Server) handleUpdateView(w http.ResponseWriter, r *http.Request) {
 			writeManagerError(w, err)
 			return
 		}
+		if reqErr := s.authorizeViewCommit(r.Context(), bucketARN, namespace, viewName, identityName); reqErr != nil {
+			writeError(w, reqErr.status, reqErr.errType, reqErr.message)
+			return
+		}
 		if getResp.Metadata == nil || len(getResp.Metadata.FullMetadata) == 0 {
 			writeError(w, http.StatusInternalServerError, "InternalServerError", "view has no metadata")
 			return

weed/s3api/iceberg/iceberg_commit_authorize_test.go

@@ -0,0 +1,237 @@
+package iceberg
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/apache/iceberg-go"
+	"github.com/apache/iceberg-go/table"
+	"github.com/apache/iceberg-go/view"
+	"github.com/gorilla/mux"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"google.golang.org/grpc"
+)
+
+// writeLogFiler records every file the handlers create, including files they
+// delete again before answering, which the final entry map cannot show.
+type writeLogFiler struct {
+	*memFiler
+	created []string
+}
+
+func (f *writeLogFiler) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
+	return fn(f)
+}
+
+func (f *writeLogFiler) CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
+	if !in.Entry.IsDirectory {
+		f.created = append(f.created, path.Join(in.Directory, in.Entry.Name))
+	}
+	return f.memFiler.CreateEntry(ctx, in, opts...)
+}
+
+// seedOwnedTable registers an existing table owned by owner.
+func seedOwnedTable(fc *memFiler, bucket, namespace, tableName, owner string) {
+	internal := map[string]any{
+		"name":             tableName,
+		"namespace":        namespace,
+		"format":           "ICEBERG",
+		"ownerAccountId":   owner,
+		"versionToken":     "v1",
+		"metadataVersion":  1,
+		"metadataLocation": "s3://" + bucket + "/" + namespace + "/" + tableName + "/metadata/v1.metadata.json",
+		"metadata": s3tables.TableMetadata{
+			Iceberg: &s3tables.IcebergMetadata{TableUUID: "00000000-0000-0000-0000-000000000001"},
+		},
+	}
+	metaBytes, _ := json.Marshal(internal)
+	fc.seed(s3tables.GetTablePath(bucket, namespace, tableName), &filer_pb.Entry{
+		Name:        tableName,
+		IsDirectory: true,
+		Extended:    map[string][]byte{s3tables.ExtendedKeyMetadata: metaBytes},
+	})
+}
+
+// seedOwnedView registers an existing view owned by owner.
+func seedOwnedView(fc *memFiler, bucket, namespace, viewName, owner string, fullMetadata []byte) {
+	internal := map[string]any{
+		"name":             viewName,
+		"namespace":        namespace,
+		"format":           "ICEBERG",
+		"ownerAccountId":   owner,
+		"versionToken":     "v1",
+		"metadataVersion":  1,
+		"metadataLocation": "s3://" + bucket + "/" + namespace + "/" + viewName + "/metadata/v1.metadata.json",
+	}
+	if len(fullMetadata) > 0 {
+		internal["metadata"] = s3tables.TableMetadata{FullMetadata: fullMetadata}
+	}
+	metaBytes, _ := json.Marshal(internal)
+	fc.seed(s3tables.GetTablePath(bucket, namespace, viewName), &filer_pb.Entry{
+		Name:        viewName,
+		IsDirectory: true,
+		Extended: map[string][]byte{
+			s3tables.ExtendedKeyMetadata:  metaBytes,
+			s3tables.ExtendedKeyEntryType: []byte(s3tables.EntryTypeView),
+		},
+	})
+}
+
+// withActions gives the request an identity allowed only the listed actions.
+func withActions(r *http.Request, name string, actions ...string) *http.Request {
+	ctx := s3_constants.SetIdentityNameInContext(r.Context(), name)
+	return r.WithContext(s3_constants.SetIdentityInContext(ctx, &struct {
+		Name    string
+		Actions []string
+	}{Name: name, Actions: actions}))
+}
+
+func viewMetadataJSON(t *testing.T, location string) []byte {
+	t.Helper()
+	schema := iceberg.NewSchemaWithIdentifiers(0, nil,
+		iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true})
+	ver, err := view.NewVersionFromSQL(1, schema.ID, "SELECT 1", table.Identifier{"finance"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	md, err := view.NewMetadata(ver, schema, location, iceberg.Properties{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	fullMetadata, err := json.Marshal(md)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return fullMetadata
+}
+
+// A caller that may load a table but not commit to it must be refused before
+// the commit stages its metadata file in the table's directory. The commit
+// paths used to write that file, reach the UpdateTable authorization, and
+// delete the file again on the way out.
+func TestCommitDeniedBeforeStagingMetadata(t *testing.T) {
+	const bucket = "warehouse"
+	cases := map[string]func(s *Server) *httptest.ResponseRecorder{
+		"commit-table": func(s *Server) *httptest.ResponseRecorder {
+			r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/tables/quarterly_reports",
+				strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`))
+			r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "table": "quarterly_reports"})
+			w := httptest.NewRecorder()
+			s.handleUpdateTable(w, withActions(r, "reader", "s3tables:GetTable"))
+			return w
+		},
+		"commit-transaction": func(s *Server) *httptest.ResponseRecorder {
+			body := `{"table-changes":[{"identifier":{"namespace":["finance"],"name":"quarterly_reports"},"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}]}`
+			r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/transactions/commit", strings.NewReader(body))
+			r = mux.SetURLVars(r, map[string]string{"prefix": bucket})
+			w := httptest.NewRecorder()
+			s.handleCommitTransaction(w, withActions(r, "reader", "s3tables:GetTable"))
+			return w
+		},
+		"commit-view": func(s *Server) *httptest.ResponseRecorder {
+			r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/views/quarterly_view",
+				strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`))
+			r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "view": "quarterly_view"})
+			w := httptest.NewRecorder()
+			s.handleUpdateView(w, withActions(r, "reader", "s3tables:GetView"))
+			return w
+		},
+	}
+	for name, run := range cases {
+		t.Run(name, func(t *testing.T) {
+			fc := &writeLogFiler{memFiler: newMemFiler()}
+			seedNamespace(fc.memFiler, bucket, "finance", "alice")
+			seedOwnedTable(fc.memFiler, bucket, "finance", "quarterly_reports", "alice")
+			seedOwnedView(fc.memFiler, bucket, "finance", "quarterly_view", "alice", viewMetadataJSON(t, "s3://"+bucket+"/finance/quarterly_view"))
+			s := NewServer(fc, nil)
+
+			w := run(s)
+
+			if w.Code != http.StatusForbidden {
+				t.Errorf("status = %d, want %d (body: %s)", w.Code, http.StatusForbidden, w.Body.String())
+			}
+			if len(fc.created) != 0 {
+				t.Errorf("refused commit wrote %v", fc.created)
+			}
+		})
+	}
+}
+
+// The owner, and a caller whose own identity carries UpdateTable, still
+// stage their metadata file and land the commit.
+func TestCommitAllowedStagesMetadata(t *testing.T) {
+	const bucket = "warehouse"
+	callers := map[string]func(r *http.Request) *http.Request{
+		"owner": func(r *http.Request) *http.Request {
+			return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
+		},
+		"identity-actions": func(r *http.Request) *http.Request {
+			return withActions(r, "writer", "s3tables:GetTable", "s3tables:UpdateTable")
+		},
+	}
+	for name, as := range callers {
+		t.Run(name, func(t *testing.T) {
+			fc := &writeLogFiler{memFiler: newMemFiler()}
+			seedNamespace(fc.memFiler, bucket, "finance", "alice")
+			seedOwnedTable(fc.memFiler, bucket, "finance", "quarterly_reports", "alice")
+			s := NewServer(fc, nil)
+
+			r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/tables/quarterly_reports",
+				strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`))
+			r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "table": "quarterly_reports"})
+			w := httptest.NewRecorder()
+			s.handleUpdateTable(w, as(r))
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusOK, w.Body.String())
+			}
+			if len(fc.created) != 1 || !strings.HasSuffix(fc.created[0], ".metadata.json") {
+				t.Fatalf("created = %v, want one metadata file", fc.created)
+			}
+		})
+	}
+}
+
+// Same staging gate on the view commit path: the owner, and a caller whose own
+// identity carries UpdateView, stage a metadata file and land the commit.
+func TestViewCommitAllowedStagesMetadata(t *testing.T) {
+	const bucket = "warehouse"
+	fullMetadata := viewMetadataJSON(t, "s3://"+bucket+"/finance/quarterly_view")
+
+	callers := map[string]func(r *http.Request) *http.Request{
+		"owner": func(r *http.Request) *http.Request {
+			return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
+		},
+		"identity-actions": func(r *http.Request) *http.Request {
+			return withActions(r, "writer", "s3tables:GetView", "s3tables:UpdateView")
+		},
+	}
+	for name, as := range callers {
+		t.Run(name, func(t *testing.T) {
+			fc := &writeLogFiler{memFiler: newMemFiler()}
+			seedNamespace(fc.memFiler, bucket, "finance", "alice")
+			seedOwnedView(fc.memFiler, bucket, "finance", "quarterly_view", "alice", fullMetadata)
+			s := NewServer(fc, nil)
+
+			r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/views/quarterly_view",
+				strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`))
+			r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "view": "quarterly_view"})
+			w := httptest.NewRecorder()
+			s.handleUpdateView(w, as(r))
+
+			if w.Code != http.StatusOK {
+				t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusOK, w.Body.String())
+			}
+			if len(fc.created) != 1 || !strings.HasSuffix(fc.created[0], ".metadata.json") {
+				t.Fatalf("created = %v, want one metadata file", fc.created)
+			}
+		})
+	}
+}

weed/s3api/iceberg/server.go

@@ -269,9 +269,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 		// The auth scheme is case-insensitive (RFC 7235), matching
 		// authenticateBearer below.
 		if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
-			if identityName, identity, ok := s.authenticateBearer(r); ok {
+			if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok {
 				ctx := r.Context()
 				ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
+				ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey)
 				if identity != nil {
 					ctx = s3_constants.SetIdentityInContext(ctx, identity)
 				}
@@ -290,6 +291,9 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 			return
 		}
 
+		// The verifier records the key of the credential that matched into
+		// this holder; the vended credentials name it.
+		r = s3_constants.EnsureIdentityHolder(r)
 		identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
 		if errCode != s3err.ErrNone {
 			apiErr := s3err.GetAPIError(errCode)

weed/s3api/iceberg/server_access_key_test.go

@@ -0,0 +1,97 @@
+package iceberg
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// keyedS3Authenticator accepts every SigV4 request as one identity and records
+// the key that verified into the request's holder, as S3ApiServer's verifiers do.
+type keyedS3Authenticator struct {
+	mockS3Authenticator
+	accessKey string
+}
+
+func (k *keyedS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) {
+	s3_constants.RecordAccessKeyId(r.Context(), k.accessKey)
+	return "shared-identity", "identity-object", s3err.ErrNone
+}
+
+// A catalog token is minted from one access key; the request it authenticates
+// carries that key, so the credentials it vends can name it.
+func TestAuthBearerRecordsTheTokensAccessKey(t *testing.T) {
+	s, _ := newAuthTestServer()
+	now := time.Now()
+	token := mintTestToken(t, "AKID123", "secret456", now, now.Add(time.Hour))
+
+	var gotKey, gotName string
+	handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
+		gotKey = s3_constants.GetAccessKeyIdFromContext(r)
+		gotName = s3_constants.GetIdentityNameFromContext(r)
+		w.WriteHeader(http.StatusOK)
+	})
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	rec := httptest.NewRecorder()
+	handler(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("valid Bearer: status = %d, want 200", rec.Code)
+	}
+	if gotName != "testuser" {
+		t.Errorf("identity = %q, want testuser", gotName)
+	}
+	if gotKey != "AKID123" {
+		t.Errorf("access key in context = %q, want AKID123", gotKey)
+	}
+}
+
+// A SigV4-signed catalog request carries the key its authenticator verified.
+func TestAuthSigV4RecordsTheAuthenticatorsAccessKey(t *testing.T) {
+	s := newTestServerWithOAuth()
+	s.authenticator = &keyedS3Authenticator{accessKey: "AKIDSIGV4"}
+
+	var gotKey string
+	handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
+		gotKey = s3_constants.GetAccessKeyIdFromContext(r)
+		w.WriteHeader(http.StatusOK)
+	})
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIDSIGV4/20261016/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=00")
+	rec := httptest.NewRecorder()
+	handler(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+	if gotKey != "AKIDSIGV4" {
+		t.Errorf("access key in context = %q, want AKIDSIGV4", gotKey)
+	}
+}
+
+// An authenticator that records no key leaves the context without one.
+func TestAuthSigV4WithoutKeyAuthenticatorRecordsNoKey(t *testing.T) {
+	s := newTestServerWithOAuth()
+	s.authenticator = &mockS3Authenticator{}
+
+	gotKey := "unset"
+	handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
+		gotKey = s3_constants.GetAccessKeyIdFromContext(r)
+		w.WriteHeader(http.StatusOK)
+	})
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil)
+	rec := httptest.NewRecorder()
+	handler(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+	if gotKey != "" {
+		t.Errorf("access key in context = %q, want none", gotKey)
+	}
+}

weed/s3api/lance/handlers_oauth.go

@@ -147,20 +147,27 @@ func mintToken(identityName, accessKey, secret string, ttlSeconds int) (string,
 // authenticateBearer validates a Bearer token from the Authorization header.
 // Returns the identity name, identity object, and whether auth succeeded.
 func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) {
+	identityName, identity, _, ok := s.authenticateBearerKey(r)
+	return identityName, identity, ok
+}
+
+// authenticateBearerKey is authenticateBearer that also returns the access key
+// the token was minted from (its verified access_key claim).
+func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) {
 	auth := r.Header.Get("Authorization")
 	if auth == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 	if !strings.HasPrefix(strings.ToLower(auth), "bearer ") {
-		return "", nil, false
+		return "", nil, "", false
 	}
 	tokenString := strings.TrimSpace(auth[7:])
 	if tokenString == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	if s.credentialValidator == nil {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	// Parse the token without verification first to get the access key,
@@ -170,17 +177,17 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool)
 	_, _, err := parser.ParseUnverified(tokenString, unverified)
 	if err != nil {
 		glog.V(2).Infof("Lance OAuth: failed to parse token: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	if unverified.AccessKey == "" {
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey)
 	if err != nil {
 		glog.V(2).Infof("Lance OAuth: failed to get credential for access key: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
 	signingKey := deriveSigningKey(unverified.AccessKey, secretKey)
@@ -193,10 +200,12 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool)
 	})
 	if err != nil || !verified.Valid {
 		glog.V(2).Infof("Lance OAuth: token verification failed: %v", err)
-		return "", nil, false
+		return "", nil, "", false
 	}
 
-	return identityName, identity, true
+	// The token's signing key is derived from this access key, so a verified
+	// token was minted from it.
+	return identityName, identity, unverified.AccessKey, true
 }
 
 // authenticateApiKey validates an x-api-key header carrying an S3 credential

weed/s3api/lance/server.go

@@ -177,9 +177,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 		// different error that clients cannot act on.
 		// The auth scheme is case-insensitive (RFC 7235).
 		if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
-			if identityName, identity, ok := s.authenticateBearer(r); ok {
+			if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok {
 				ctx := r.Context()
 				ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
+				ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey)
 				if identity != nil {
 					ctx = s3_constants.SetIdentityInContext(ctx, identity)
 				}
@@ -194,8 +195,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 
 		if apiKey := r.Header.Get("x-api-key"); apiKey != "" {
 			if identityName, identity, ok := s.authenticateApiKey(apiKey); ok {
+				accessKey, _, _ := strings.Cut(apiKey, ":")
 				ctx := r.Context()
 				ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
+				ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey)
 				if identity != nil {
 					ctx = s3_constants.SetIdentityInContext(ctx, identity)
 				}
@@ -212,6 +215,9 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
 			return
 		}
 
+		// The verifier records the key of the credential that matched into
+		// this holder; the vended credentials name it.
+		r = s3_constants.EnsureIdentityHolder(r)
 		identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
 		if errCode != s3err.ErrNone {
 			apiErr := s3err.GetAPIError(errCode)

weed/s3api/lance/server_access_key_test.go

@@ -0,0 +1,69 @@
+package lance
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+// keyedS3Authenticator accepts every request as one identity and records the
+// key that verified into the request's holder, as S3ApiServer's verifiers do.
+type keyedS3Authenticator struct {
+	mockS3Authenticator
+	accessKey string
+}
+
+func (k *keyedS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) {
+	s3_constants.RecordAccessKeyId(r.Context(), k.accessKey)
+	return "s3user", nil, s3err.ErrNone
+}
+
+func accessKeySeenByHandler(t *testing.T, s *Server, req *http.Request) string {
+	t.Helper()
+	gotKey := "unset"
+	handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
+		gotKey = s3_constants.GetAccessKeyIdFromContext(r)
+		w.WriteHeader(http.StatusOK)
+	})
+	rec := httptest.NewRecorder()
+	handler(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+	return gotKey
+}
+
+// Each way a Lance client authenticates leaves the access key it used in the
+// request context, so the credentials it vends can name that key.
+func TestAuthRecordsTheCallersAccessKey(t *testing.T) {
+	now := time.Now()
+
+	s := newTestServerWithOAuth()
+	req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
+	req.Header.Set("Authorization", "Bearer "+mintTestToken(t, "AKID123", "secret456", now, now.Add(time.Hour)))
+	if got := accessKeySeenByHandler(t, s, req); got != "AKID123" {
+		t.Errorf("Bearer: access key in context = %q, want AKID123", got)
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
+	req.Header.Set("x-api-key", "AKID123:secret456")
+	if got := accessKeySeenByHandler(t, s, req); got != "AKID123" {
+		t.Errorf("x-api-key: access key in context = %q, want AKID123", got)
+	}
+
+	s.authenticator = &keyedS3Authenticator{accessKey: "AKIDSIGV4"}
+	req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
+	if got := accessKeySeenByHandler(t, s, req); got != "AKIDSIGV4" {
+		t.Errorf("SigV4: access key in context = %q, want AKIDSIGV4", got)
+	}
+
+	s.authenticator = &mockS3Authenticator{}
+	req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
+	if got := accessKeySeenByHandler(t, s, req); got != "" {
+		t.Errorf("authenticator without a key: access key in context = %q, want none", got)
+	}
+}

weed/s3api/policy_engine/README_POLICY_ENGINE.md

@@ -356,6 +356,7 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim
 | `aws:SecureTransport` | Require HTTPS |
 | `aws:CurrentTime` | Time-based access control |
 | `s3:ExistingObjectTag/<key>` | Tag-based access control for existing objects |
+| `s3:RequestObjectTag/<key>` | Tag conditions on the tags a PUT request sets |
 | Wildcard Patterns | Support for `*` and `?` in actions and resources |
 | Principal Matching | `*`, account IDs, and user ARNs |
 
@@ -363,7 +364,6 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim
 
 | Feature | GitHub Issue |
 |---------|--------------|
-| `s3:RequestObjectTag/<key>` | For tag conditions on PUT requests |
 | `s3:RequestObjectTagKeys` | Check which tag keys are in request |
 | `s3:x-amz-content-sha256` | Content hash condition |
 | `s3:x-amz-server-side-encryption` | SSE condition |

weed/s3api/policy_engine/conditions.go

@@ -728,6 +728,10 @@ func GetConditionEvaluator(operator string) (ConditionEvaluator, error) {
 // ExistingObjectTagPrefix is the prefix for S3 policy condition keys
 const ExistingObjectTagPrefix = "s3:ExistingObjectTag/"
 
+// RequestObjectTagPrefix is the prefix for condition keys on the tags a
+// request sets through the X-Amz-Tagging header.
+const RequestObjectTagPrefix = "s3:RequestObjectTag/"
+
 // getConditionContextValue resolves the value(s) for a condition key.
 // For s3:ExistingObjectTag/<key> conditions, it looks up the tag in objectEntry.
 // For other condition keys, it looks up the value in contextValues.

weed/s3api/policy_engine/engine.go

@@ -3,6 +3,7 @@ package policy_engine
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 	"sync"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )
 
 // PolicyEvaluationResult represents the result of policy evaluation
@@ -439,6 +441,10 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m
 	// Note: s3:ExistingObjectTag/<key> conditions are evaluated using objectEntry
 	// passed to EvaluatePolicy, not extracted from the request.
 
+	for key, value := range requestObjectTags(r) {
+		values[RequestObjectTagPrefix+key] = []string{value}
+	}
+
 	// S3 bucket-level conditions
 	if delimiter := r.URL.Query().Get("delimiter"); delimiter != "" {
 		values["s3:delimiter"] = []string{delimiter}
@@ -496,6 +502,86 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m
 	return values
 }
 
+// requestObjectTags returns the tags that X-Amz-Tagging will store on the
+// object, or nil when the handler ignores the header — otherwise a tag
+// attached to a read or to a write that never stores it would satisfy a
+// tag-conditioned policy the object cannot meet.
+func requestObjectTags(r *http.Request) map[string]string {
+	tagging := r.Header.Get(s3_constants.AmzObjectTagging)
+	if tagging == "" {
+		return nil
+	}
+	if !storesRequestObjectTags(r) {
+		return nil
+	}
+	// A malformed header is rejected by the handler with InvalidTag.
+	parsed, err := url.ParseQuery(tagging)
+	if err != nil {
+		return nil
+	}
+	// putToFiler rejects a repeated key, but CopyObject stores its last
+	// value, so that is the one the policy has to see.
+	tags := make(map[string]string, len(parsed))
+	for key, vals := range parsed {
+		tags[key] = vals[len(vals)-1]
+	}
+	return tags
+}
+
+// storesRequestObjectTags reports whether the request's handler saves the
+// X-Amz-Tagging header on the object: PutObject, a copy under
+// X-Amz-Tagging-Directive: REPLACE, and CreateMultipartUpload do; every
+// other operation — reads, deletes, part uploads, completion, tagging
+// subresources — does not.
+func storesRequestObjectTags(r *http.Request) bool {
+	query := r.URL.Query()
+	if r.Method == http.MethodPost {
+		return query.Has("uploads") && !query.Has("uploadId")
+	}
+	if r.Method != http.MethodPut {
+		return false
+	}
+	// A numeric partNumber together with uploadId routes to the part
+	// handlers — with or without a copy source — which never read
+	// X-Amz-Tagging.
+	if query.Has("partNumber") && query.Has("uploadId") && isDigits(query.Get("partNumber")) {
+		return false
+	}
+	// The PUT subresources route before CopyObject and never read
+	// X-Amz-Tagging; any other query key — presign parameters included —
+	// still reaches the object PUT handlers.
+	for _, subresource := range []string{"tagging", "renameObject", "acl", "retention", "legal-hold"} {
+		if query.Has(subresource) {
+			return false
+		}
+	}
+	if r.Header.Get("X-Amz-Copy-Source") != "" {
+		return r.Header.Get(s3_constants.AmzObjectTaggingDirective) == "REPLACE"
+	}
+	if query.Has("partNumber") || query.Has("uploadId") {
+		return false
+	}
+	// An explicit directory-marker PUT stores no tags.
+	if strings.HasSuffix(r.URL.Path, "/") && r.ContentLength <= 1024 {
+		return false
+	}
+	return true
+}
+
+// isDigits reports whether s is non-empty and all ASCII digits, matching the
+// partNumber route constraint.
+func isDigits(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, c := range s {
+		if c < '0' || c > '9' {
+			return false
+		}
+	}
+	return true
+}
+
 // IsMultipartContinuationAction returns true for actions that do not carry
 // their own SSE header because SSE is inherited from CreateMultipartUpload.
 func IsMultipartContinuationAction(action string) bool {

weed/s3api/policy_engine/engine_test.go

@@ -896,6 +896,161 @@ func TestExistingObjectTagCondition(t *testing.T) {
 	}
 }
 
+func TestRequestObjectTagCondition(t *testing.T) {
+	engine := NewPolicyEngine()
+
+	policyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:PutObject",
+				"Resource": "arn:aws:s3:::test-bucket/*",
+				"Condition": {
+					"StringEquals": {
+						"s3:RequestObjectTag/department": ["engineering"]
+					}
+				}
+			}
+		]
+	}`
+
+	if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil {
+		t.Fatalf("Failed to set bucket policy: %v", err)
+	}
+
+	copyHeaders := map[string]string{"X-Amz-Copy-Source": "/src-bucket/src-object"}
+	replaceHeaders := map[string]string{
+		"X-Amz-Copy-Source":                    "/src-bucket/src-object",
+		s3_constants.AmzObjectTaggingDirective: "REPLACE",
+	}
+
+	tests := []struct {
+		name     string
+		method   string
+		target   string
+		headers  map[string]string
+		tagging  string
+		expected PolicyEvaluationResult
+	}{
+		{"matching tag allows", "", "", nil, "department=engineering", PolicyResultAllow},
+		{"matching tag among others allows", "", "", nil, "project=alpha&department=engineering", PolicyResultAllow},
+		{"url-encoded value is decoded", "", "", nil, "department=engin%65ering", PolicyResultAllow},
+		{"other value is not allowed", "", "", nil, "department=sales", PolicyResultIndeterminate},
+		{"missing tag is not allowed", "", "", nil, "project=alpha", PolicyResultIndeterminate},
+		{"no tagging header is not allowed", "", "", nil, "", PolicyResultIndeterminate},
+		{"malformed header is not allowed", "", "", nil, "department=engineering&%zz", PolicyResultIndeterminate},
+		{"last value of a repeated key counts", "", "", nil, "department=sales&department=engineering", PolicyResultAllow},
+		{"earlier value of a repeated key is ignored", "", "", nil, "department=engineering&department=sales", PolicyResultIndeterminate},
+		{"header is ignored on PutObjectTagging", "", "?tagging", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header counts with an unrelated query parameter", "", "?foo=bar", nil, "department=engineering", PolicyResultAllow},
+		{"header counts on a Signature V2 presigned PUT", "", "?AWSAccessKeyId=k&Signature=s&Expires=1", nil, "department=engineering", PolicyResultAllow},
+		{"header is ignored on a renameObject subresource", "", "?renameObject", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on a copy that keeps source tags", "", "", copyHeaders, "department=engineering", PolicyResultIndeterminate},
+		{"header counts on a copy that replaces tags", "", "", replaceHeaders, "department=engineering", PolicyResultAllow},
+		{"header is ignored on a multipart copy that replaces tags", "", "?partNumber=1&uploadId=x", replaceHeaders, "department=engineering", PolicyResultIndeterminate},
+		{"header counts on a copy with a lone partNumber", "", "?partNumber=1", replaceHeaders, "department=engineering", PolicyResultAllow},
+		{"header counts on a multipart create", "POST", "?uploads", nil, "department=engineering", PolicyResultAllow},
+		{"header is ignored on a part upload", "", "?partNumber=1&uploadId=x", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on multipart completion", "POST", "?uploadId=x", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on a directory marker", "", "/", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on an acl subresource", "", "?acl", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on a GET", "GET", "", nil, "department=engineering", PolicyResultIndeterminate},
+		{"header is ignored on a DELETE", "DELETE", "", nil, "department=engineering", PolicyResultIndeterminate},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			method := tt.method
+			if method == "" {
+				method = http.MethodPut
+			}
+			req := httptest.NewRequest(method, "/test-bucket/test-object"+tt.target, nil)
+			for k, v := range tt.headers {
+				req.Header.Set(k, v)
+			}
+			if tt.tagging != "" {
+				req.Header.Set(s3_constants.AmzObjectTagging, tt.tagging)
+			}
+
+			args := &PolicyEvaluationArgs{
+				Action:     "s3:PutObject",
+				Resource:   "arn:aws:s3:::test-bucket/test-object",
+				Principal:  "*",
+				Conditions: engine.ExtractConditionValuesFromRequest(req),
+			}
+
+			if result := engine.EvaluatePolicy("test-bucket", args); result != tt.expected {
+				t.Errorf("Expected %v, got %v", tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestRequestObjectTagConditionDeny(t *testing.T) {
+	engine := NewPolicyEngine()
+
+	policyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:PutObject",
+				"Resource": "arn:aws:s3:::test-bucket/*"
+			},
+			{
+				"Effect": "Deny",
+				"Principal": "*",
+				"Action": "s3:PutObject",
+				"Resource": "arn:aws:s3:::test-bucket/*",
+				"Condition": {
+					"StringNotEquals": {
+						"s3:RequestObjectTag/department": ["engineering"]
+					}
+				}
+			}
+		]
+	}`
+
+	if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil {
+		t.Fatalf("Failed to set bucket policy: %v", err)
+	}
+
+	tests := []struct {
+		name     string
+		target   string
+		tagging  string
+		expected PolicyEvaluationResult
+	}{
+		{"required tag passes", "", "department=engineering", PolicyResultAllow},
+		{"other value is denied", "", "department=sales", PolicyResultDeny},
+		{"missing tag is denied", "", "project=alpha", PolicyResultDeny},
+		{"repeated key ending in another value is denied", "", "department=engineering&department=sales", PolicyResultDeny},
+		{"malformed header is denied", "", "department=engineering&%zz", PolicyResultDeny},
+		{"unrelated query parameter does not hide the tag", "?foo=bar", "department=sales", PolicyResultDeny},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodPut, "/test-bucket/test-object"+tt.target, nil)
+			req.Header.Set(s3_constants.AmzObjectTagging, tt.tagging)
+
+			args := &PolicyEvaluationArgs{
+				Action:     "s3:PutObject",
+				Resource:   "arn:aws:s3:::test-bucket/test-object",
+				Principal:  "*",
+				Conditions: engine.ExtractConditionValuesFromRequest(req),
+			}
+
+			if result := engine.EvaluatePolicy("test-bucket", args); result != tt.expected {
+				t.Errorf("Expected %v, got %v", tt.expected, result)
+			}
+		})
+	}
+}
+
 // TestExistingObjectTagConditionMultipleTags tests policies with multiple tag conditions
 func TestExistingObjectTagConditionMultipleTags(t *testing.T) {
 	engine := NewPolicyEngine()

weed/s3api/s3_action_resolver.go

@@ -166,15 +166,25 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool)
 		}
 	}
 
-	// ACL operations
+	// ACL operations. A versionId retargets the request at a specific
+	// version, so the version ACL action applies, not the current-object one.
 	if query.Has("acl") {
+		versioned := hasObject && query.Get("versionId") != ""
 		switch method {
-		case http.MethodGet, http.MethodHead:
+		// HEAD has no acl route; it lands on HeadObjectHandler and should
+		// resolve like a read — s3:GetObject, or s3:GetObjectVersion below.
+		case http.MethodGet:
+			if versioned {
+				return s3_constants.S3_ACTION_GET_OBJECT_VERSION_ACL
+			}
 			if hasObject {
 				return s3_constants.S3_ACTION_GET_OBJECT_ACL
 			}
 			return s3_constants.S3_ACTION_GET_BUCKET_ACL
 		case http.MethodPut:
+			if versioned {
+				return s3_constants.S3_ACTION_PUT_OBJECT_VERSION_ACL
+			}
 			if hasObject {
 				return s3_constants.S3_ACTION_PUT_OBJECT_ACL
 			}
@@ -220,6 +230,9 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool)
 	// Must be checked before versionId, because GET /bucket/key?attributes&versionId=xyz
 	// is a GetObjectAttributes request, not a GetObjectVersion request
 	if hasObject && query.Has("attributes") && method == http.MethodGet {
+		if query.Get("versionId") != "" {
+			return s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES
+		}
 		return s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES
 	}
 

weed/s3api/s3_action_resolver_test.go

@@ -83,7 +83,7 @@ func TestResolveS3Action_AttributesBeforeVersionId(t *testing.T) {
 			method:     http.MethodGet,
 			baseAction: s3_constants.ACTION_READ,
 			object:     "key",
-			want:       s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES,
+			want:       s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES,
 		},
 		{
 			name:       "versionId only GET",
@@ -146,6 +146,59 @@ func TestResolveS3Action_VersionTagging(t *testing.T) {
 	}
 }
 
+// An ACL request carrying versionId targets a specific version and must
+// resolve to the version ACL action so a grant limited to the current-object
+// action cannot reach historical versions.
+func TestResolveS3Action_VersionAcl(t *testing.T) {
+	tests := []struct {
+		name   string
+		method string
+		object string
+		query  string
+		want   string
+	}{
+		{"get current acl", http.MethodGet, "key", "acl", s3_constants.S3_ACTION_GET_OBJECT_ACL},
+		{"put current acl", http.MethodPut, "key", "acl", s3_constants.S3_ACTION_PUT_OBJECT_ACL},
+		{"get version acl", http.MethodGet, "key", "acl&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_ACL},
+		{"head acl resolves as head object", http.MethodHead, "key", "acl", s3_constants.S3_ACTION_GET_OBJECT},
+		{"head version acl resolves as head version", http.MethodHead, "key", "acl&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION},
+		{"put version acl", http.MethodPut, "key", "acl&versionId=v1", s3_constants.S3_ACTION_PUT_OBJECT_VERSION_ACL},
+		{"empty versionId stays current", http.MethodGet, "key", "acl&versionId=", s3_constants.S3_ACTION_GET_OBJECT_ACL},
+		{"bucket acl ignores versionId", http.MethodGet, "", "acl&versionId=v1", s3_constants.S3_ACTION_GET_BUCKET_ACL},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil)
+			if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want {
+				t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+// An attributes request carrying versionId targets a specific version and
+// must resolve to the version attributes action.
+func TestResolveS3Action_VersionAttributes(t *testing.T) {
+	tests := []struct {
+		name   string
+		method string
+		object string
+		query  string
+		want   string
+	}{
+		{"get current attributes", http.MethodGet, "key", "attributes", s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES},
+		{"get version attributes", http.MethodGet, "key", "attributes&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil)
+			if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want {
+				t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
 // Bucket subresources registered with ACTION_ADMIN must resolve to their own
 // S3 actions so a policy granting one of them does not need s3:*, and so no
 // broader grant sweeps them in.

weed/s3api/s3_constants/header.go

@@ -216,10 +216,21 @@ func GetBucketAndObject(r *http.Request) (bucket, object string) {
 	return
 }
 
-// IsValidObjectKey rejects S3 object keys that — after normalization
-// (backslash→slash, slash collapse) — contain a `.` or `..` path segment, or
-// embed a NUL byte. Such keys are collapsed by filepath.Join inside the filer
-// and would escape the bucket directory, so they must not reach the gRPC layer.
+// allowBackslashInObjectKey keeps '\' a literal object-key character instead
+// of normalizing it to '/'. It is a process-wide mode set once at S3 server
+// startup; the filer only treats '/' as a path separator, so a verbatim '\'
+// stays inside one filer entry name.
+var allowBackslashInObjectKey atomic.Bool
+
+func SetAllowBackslashInObjectKey(allow bool) {
+	allowBackslashInObjectKey.Store(allow)
+}
+
+// IsValidObjectKey rejects S3 object keys that — after the mode-dependent
+// normalization NormalizeObjectKey applies — contain a `.` or `..` path
+// segment, or embed a NUL byte. Such keys are collapsed by filepath.Join
+// inside the filer and would escape the bucket directory, so they must not
+// reach the gRPC layer.
 // Gorilla mux URL-decodes captured vars before this runs, so `%2e%2e` is
 // already `..` here.
 func IsValidObjectKey(object string) bool {
@@ -229,7 +240,9 @@ func IsValidObjectKey(object string) bool {
 	if strings.ContainsRune(object, '\x00') {
 		return false
 	}
-	object = strings.ReplaceAll(object, "\\", "/")
+	if !allowBackslashInObjectKey.Load() {
+		object = strings.ReplaceAll(object, "\\", "/")
+	}
 	for _, seg := range strings.Split(object, "/") {
 		if seg == "." || seg == ".." {
 			return false
@@ -263,7 +276,8 @@ func IsValidPathSegment(value string) bool {
 
 // NormalizeObjectKey normalizes object keys by removing duplicate slashes and converting backslashes.
 // This normalizes keys from various sources (URL path, form values, etc.) to a consistent format.
-// It also converts Windows-style backslashes to forward slashes for cross-platform compatibility.
+// It also converts Windows-style backslashes to forward slashes for cross-platform compatibility,
+// unless allowBackslashInObjectKey is set.
 // Returns keys WITHOUT leading slash to match S3 API format (e.g., "foo/bar" not "/foo/bar").
 // Preserves trailing slash if present (e.g., "foo/" stays "foo/").
 func NormalizeObjectKey(object string) string {
@@ -271,7 +285,9 @@ func NormalizeObjectKey(object string) string {
 	hasTrailingSlash := strings.HasSuffix(object, "/")
 
 	// Convert Windows-style backslashes to forward slashes
-	object = strings.ReplaceAll(object, "\\", "/")
+	if !allowBackslashInObjectKey.Load() {
+		object = strings.ReplaceAll(object, "\\", "/")
+	}
 	object = removeDuplicateSlashes(object)
 	// Remove leading slash to match S3 API format
 	object = strings.TrimPrefix(object, "/")
@@ -338,6 +354,7 @@ const (
 	contextKeyIdentityClaim       contextKey = "s3-identity-claim"
 	contextKeyIgnoredSessionToken contextKey = "s3-ignored-session-token"
 	contextKeyOriginalRequestPath contextKey = "s3-original-request-path"
+	contextKeyAccessKeyId         contextKey = "s3-access-key-id"
 )
 
 // identityHolder is a mutable container for the authenticated identity name,
@@ -351,6 +368,7 @@ type identityHolder struct {
 	name          atomic.Pointer[string]
 	principalArn  atomic.Pointer[string]
 	identityClaim atomic.Pointer[string]
+	accessKeyId   atomic.Pointer[string]
 }
 
 // EnsureIdentityHolder attaches a mutable identity holder to the request context
@@ -480,6 +498,62 @@ func GetOriginalRequestPathFromContext(r *http.Request) string {
 	return ""
 }
 
+// SetAccessKeyIdInContext stores the access key id the request authenticated
+// with: a static key's id, or an STS session's temporary key id. Several keys
+// can belong to one identity, so the identity name alone cannot tell their
+// holders apart in the audit log. An access key id is not a secret (AWS logs
+// it in CloudTrail); the secret key never reaches the context.
+func SetAccessKeyIdInContext(ctx context.Context, accessKeyId string) context.Context {
+	if accessKeyId == "" {
+		return ctx
+	}
+	if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil {
+		h.accessKeyId.Store(&accessKeyId)
+	}
+	return context.WithValue(ctx, contextKeyAccessKeyId, accessKeyId)
+}
+
+// RecordAccessKeyId stores the access key id of a credential whose signature
+// was just verified into the request's identity holder, so the outer
+// middleware (the audit fallback) and the handler both see it. A no-op without
+// a holder (EnsureIdentityHolder) or without a key. Only a verifier calls it:
+// the key comes from the credential that matched, never from re-reading the
+// request, whose shape the verifier already resolved.
+func RecordAccessKeyId(ctx context.Context, accessKeyId string) {
+	if ctx == nil || accessKeyId == "" {
+		return
+	}
+	if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil {
+		h.accessKeyId.Store(&accessKeyId)
+	}
+}
+
+// GetAccessKeyIdFromContext retrieves the access key id the request
+// authenticated with, or "" when it carried none (anonymous, OIDC bearer).
+func GetAccessKeyIdFromContext(r *http.Request) string {
+	if r == nil {
+		return ""
+	}
+	return AccessKeyIdFromContext(r.Context())
+}
+
+// AccessKeyIdFromContext is GetAccessKeyIdFromContext for code that holds
+// only the request's context, such as the Iceberg credential vendor.
+func AccessKeyIdFromContext(ctx context.Context) string {
+	if ctx == nil {
+		return ""
+	}
+	if id, ok := ctx.Value(contextKeyAccessKeyId).(string); ok && id != "" {
+		return id
+	}
+	if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil {
+		if id := h.accessKeyId.Load(); id != nil {
+			return *id
+		}
+	}
+	return ""
+}
+
 // IgnoreSessionTokenInContext marks the request's session token as tolerated
 // rather than authenticated: a statically configured access key signed the
 // request, so the token (which credential vendors like Unity Catalog emit on

weed/s3api/s3_constants/header_test.go

@@ -123,6 +123,54 @@ func TestIsValidObjectKey(t *testing.T) {
 	}
 }
 
+func TestNormalizeObjectKeyAllowBackslash(t *testing.T) {
+	SetAllowBackslashInObjectKey(true)
+	defer SetAllowBackslashInObjectKey(false)
+
+	normalizeTests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"backslash kept verbatim", "folder\\file.txt", "folder\\file.txt"},
+		{"nested backslash kept", "folder\\sub\\file.txt", "folder\\sub\\file.txt"},
+		{"trailing backslash kept", "folder\\", "folder\\"},
+		{"lone backslash kept", "\\", "\\"},
+		{"double backslash kept", "a\\\\b", "a\\\\b"},
+		{"mixed separators", "folder\\sub/another\\file.txt", "folder\\sub/another\\file.txt"},
+		{"duplicate slashes still collapse", "folder//sub///file.txt", "folder/sub/file.txt"},
+		{"leading slash still trimmed", "/folder\\file.txt", "folder\\file.txt"},
+		{"trailing slash preserved", "folder/", "folder/"},
+		{"windows path style", "C:\\Users\\test\\file.txt", "C:\\Users\\test\\file.txt"},
+	}
+	for _, tt := range normalizeTests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := NormalizeObjectKey(tt.input); got != tt.expected {
+				t.Errorf("NormalizeObjectKey(%q) = %q, want %q", tt.input, got, tt.expected)
+			}
+		})
+	}
+
+	validTests := []struct {
+		name  string
+		input string
+		want  bool
+	}{
+		{"backslash dotdot is one segment", "..\\evil\\test.txt", true},
+		{"backslash dotdot inside", "good\\..\\evil/test.txt", true},
+		{"nested dotdot still rejected", "good/../evil/test.txt", false},
+		{"leading dotdot still rejected", "../evil/test.txt", false},
+		{"plain backslash key", "folder\\file.txt", true},
+	}
+	for _, tt := range validTests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsValidObjectKey(tt.input); got != tt.want {
+				t.Errorf("IsValidObjectKey(%q) = %v, want %v", tt.input, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestIsValidBucketName(t *testing.T) {
 	tests := []struct {
 		name  string

weed/s3api/s3_constants/s3_action_strings.go

@@ -4,17 +4,22 @@ package s3_constants
 // These match the official AWS S3 action format used in IAM and bucket policies
 const (
 	// Object operations
-	S3_ACTION_GET_OBJECT            = "s3:GetObject"
-	S3_ACTION_PUT_OBJECT            = "s3:PutObject"
-	S3_ACTION_DELETE_OBJECT         = "s3:DeleteObject"
-	S3_ACTION_DELETE_OBJECT_VERSION = "s3:DeleteObjectVersion"
-	S3_ACTION_GET_OBJECT_VERSION    = "s3:GetObjectVersion"
-	S3_ACTION_GET_OBJECT_ATTRIBUTES = "s3:GetObjectAttributes"
+	S3_ACTION_GET_OBJECT                    = "s3:GetObject"
+	S3_ACTION_PUT_OBJECT                    = "s3:PutObject"
+	S3_ACTION_DELETE_OBJECT                 = "s3:DeleteObject"
+	S3_ACTION_DELETE_OBJECT_VERSION         = "s3:DeleteObjectVersion"
+	S3_ACTION_GET_OBJECT_VERSION            = "s3:GetObjectVersion"
+	S3_ACTION_GET_OBJECT_ATTRIBUTES         = "s3:GetObjectAttributes"
+	S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES = "s3:GetObjectVersionAttributes"
 
 	// Object ACL operations
 	S3_ACTION_GET_OBJECT_ACL = "s3:GetObjectAcl"
 	S3_ACTION_PUT_OBJECT_ACL = "s3:PutObjectAcl"
 
+	// Version-specific ACL operations (?acl&versionId)
+	S3_ACTION_GET_OBJECT_VERSION_ACL = "s3:GetObjectVersionAcl"
+	S3_ACTION_PUT_OBJECT_VERSION_ACL = "s3:PutObjectVersionAcl"
+
 	// Object tagging operations
 	S3_ACTION_GET_OBJECT_TAGGING    = "s3:GetObjectTagging"
 	S3_ACTION_PUT_OBJECT_TAGGING    = "s3:PutObjectTagging"

weed/s3api/s3_constants/s3_actions.go

@@ -29,6 +29,7 @@ const (
 	SeaweedStorageDestinationHeader = "x-seaweedfs-destination"
 	MultipartUploadsFolder          = ".uploads"
 	VersionsFolder                  = ".versions"
+	SystemFolder                    = ".system"
 	FolderMimeType                  = "httpd/unix-directory"
 	DirectoryMimeType               = "application/x-directory"
 )

weed/s3api/s3_existing_object_tag_test.go

@@ -576,3 +576,97 @@ func TestRecheckObjectAclTagConditionedPolicy(t *testing.T) {
 	assert.Equal(t, s3err.ErrNone,
 		s3a.recheckPolicyWithObjectEntry(putReq, "b", "secret.txt", s3_constants.S3_ACTION_PUT_OBJECT_ACL, publicEntry, "test"))
 }
+
+// A versioned GetObjectAttributes request is authorized against the version
+// read grant, not the attributes action the original request resolves to: an
+// explicit bucket-policy Deny on s3:GetObjectVersion blocks it, while a legacy
+// identity holding only the coarse Read grant is still accepted.
+func TestCheckVersionReadGrant(t *testing.T) {
+	s3a := &S3ApiServer{
+		policyEngine: NewBucketPolicyEngine(),
+		iam:          &IdentityAccessManagement{},
+	}
+
+	denyPolicyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:*",
+				"Resource": ["arn:aws:s3:::deny-bucket", "arn:aws:s3:::deny-bucket/*"]
+			},
+			{
+				"Effect": "Deny",
+				"Principal": "*",
+				"Action": "s3:GetObjectVersion",
+				"Resource": "arn:aws:s3:::deny-bucket/key"
+			}
+		]
+	}`
+	require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("deny-bucket", denyPolicyJSON))
+
+	allowPolicyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:*",
+				"Resource": ["arn:aws:s3:::allow-bucket", "arn:aws:s3:::allow-bucket/*"]
+			}
+		]
+	}`
+	require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("allow-bucket", allowPolicyJSON))
+
+	identity := &Identity{
+		Name:         "reader",
+		Actions:      []Action{s3_constants.ACTION_READ},
+		PrincipalArn: "arn:aws:iam::000000000000:user/reader",
+	}
+
+	attrsReq := func(bucket string) *http.Request {
+		req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/key?attributes&versionId=v1", nil)
+		return req.WithContext(s3_constants.SetIdentityInContext(req.Context(), identity))
+	}
+
+	assert.Equal(t, s3err.ErrAccessDenied,
+		s3a.checkVersionReadGrant(attrsReq("deny-bucket"), "deny-bucket", "key", "v1", nil))
+	assert.Equal(t, s3err.ErrNone,
+		s3a.checkVersionReadGrant(attrsReq("allow-bucket"), "allow-bucket", "key", "v1", nil))
+	assert.Equal(t, s3err.ErrNone,
+		s3a.checkVersionReadGrant(attrsReq("no-policy-bucket"), "no-policy-bucket", "key", "v1", nil))
+}
+
+// Presigned-URL credentials ride along on the synthesized version-read
+// request, so an s3:authType=REST-QUERY-STRING policy condition decides the
+// recheck the same way it decided the real request.
+func TestCheckVersionReadGrantPreservesQueryAuth(t *testing.T) {
+	s3a := &S3ApiServer{
+		policyEngine: NewBucketPolicyEngine(),
+		iam:          &IdentityAccessManagement{},
+	}
+
+	policyJSON := `{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": "s3:*",
+			"Resource": ["arn:aws:s3:::qs-bucket", "arn:aws:s3:::qs-bucket/*"],
+			"Condition": {"StringEquals": {"s3:authType": "REST-QUERY-STRING"}}
+		}]
+	}`
+	require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("qs-bucket", policyJSON))
+
+	presigned := httptest.NewRequest(http.MethodGet,
+		"/qs-bucket/key?attributes&versionId=v1&AWSAccessKeyId=AK&Signature=sig&Expires=9999999999", nil)
+	assert.Equal(t, s3err.ErrNone,
+		s3a.checkVersionReadGrant(presigned, "qs-bucket", "key", "v1", nil))
+
+	keyReq := objectKeyPolicyRequest(presigned, http.MethodGet, "qs-bucket", "key", "v1")
+	q := keyReq.URL.Query()
+	assert.Equal(t, "AK", q.Get("AWSAccessKeyId"))
+	assert.Equal(t, "v1", q.Get("versionId"))
+	assert.False(t, q.Has("attributes"))
+}

weed/s3api/s3api_bucket_owner_index.go

@@ -27,7 +27,7 @@ import (
 // indexes pre-existing buckets. The .complete marker is written once the
 // backfill has finished; until it exists readers keep scanning /buckets.
 const (
-	s3SystemFolder              = ".system"
+	s3SystemFolder              = s3_constants.SystemFolder
 	bucketOwnerIndexFolder      = "owners"
 	bucketOwnerIndexReadyMarker = ".complete"
 )

weed/s3api/s3api_embedded_iam.go

@@ -2609,8 +2609,8 @@ func (e *EmbeddedIamApi) handleImplicitUsername(r *http.Request, values url.Valu
 	}
 }
 
-// iamSelfServiceActions are actions that users can perform on their own resources without admin rights.
-// According to AWS IAM, users can manage their own access keys without requiring full admin permissions.
+// iamSelfServiceActions are actions that users can perform on their own resources without admin rights,
+// unless selfServiceRequiresGrant is set, in which case they need an iam:<Action> grant like any other.
 var iamSelfServiceActions = map[string]bool{
 	"CreateAccessKey": true,
 	"DeleteAccessKey": true,
@@ -2656,15 +2656,18 @@ func iamTargetUserName(action string, r *http.Request) string {
 // data-plane policy, which says nothing about administering the credential
 // store.
 //
-// Users may run self-service actions against their own identity without any
-// IAM grant, matching AWS.
+// By default users may run self-service actions against their own identity
+// without any IAM grant. With selfServiceRequiresGrant set they need the
+// iam:<Action> grant as AWS requires, authorized against the target user's
+// ARN (the caller's own when UserName is empty), so a policy can confine it
+// to the caller's own keys.
 func (iam *IdentityAccessManagement) AuthorizeIamAction(r *http.Request, identity *Identity, action, targetUserName string) s3err.ErrorCode {
 	// The anonymous identity has no user of its own, so every self-service
 	// action it names would run against someone else's.
 	if identity == nil || identity.Name == s3_constants.AccountAnonymousId {
 		return s3err.ErrAccessDenied
 	}
-	if iamRequiresAdminForOthers(action) && (targetUserName == "" || targetUserName == identity.Name) {
+	if !iam.selfServiceRequiresGrant && iamRequiresAdminForOthers(action) && (targetUserName == "" || targetUserName == identity.Name) {
 		return s3err.ErrNone
 	}
 	if identity.isAdmin() {
@@ -2677,6 +2680,13 @@ func (iam *IdentityAccessManagement) AuthorizeIamAction(r *http.Request, identit
 	if iamRequiresSelfTarget(action) && targetUserName != "" {
 		resource = fmt.Sprintf("arn:aws:iam:::user/%s", targetUserName)
 	}
+	if iam.selfServiceRequiresGrant && iamRequiresAdminForOthers(action) {
+		target := targetUserName
+		if target == "" {
+			target = identity.Name
+		}
+		resource = fmt.Sprintf("arn:aws:iam:::user/%s", target)
+	}
 	return iam.VerifyActionPermission(r, identity, Action("iam:"+action), resource, "")
 }
 

weed/s3api/s3api_embedded_iam_authz_test.go

@@ -145,3 +145,75 @@ func TestAuthorizeIamActionCreateServiceAccountScopedByPolicyResource(t *testing
 	assert.Equal(t, s3err.ErrNone,
 		iam.AuthorizeIamAction(iamPostRequest(""), admin, "CreateServiceAccount", "victim"))
 }
+
+var iamSelfServiceActionNames = []string{"CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey", "ListAccessKeys", "GetUser"}
+
+func ownKeysPolicy(resource string) string {
+	return `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:UpdateAccessKey","iam:ListAccessKeys","iam:GetUser"],"Resource":["` + resource + `"]}]}`
+}
+
+// With selfServiceRequiresGrant, an identity holding only data-plane rights may
+// no longer manage its own access keys: a leaked key cannot mint further keys
+// that outlive its own revocation, nor deactivate the keys it was issued.
+func TestAuthorizeIamActionSelfServiceRequiresGrantWhenConfigured(t *testing.T) {
+	iam := newIamAuthzTestIam(t)
+	iam.selfServiceRequiresGrant = true
+	powerUser := &Identity{Name: "power_user", PolicyNames: []string{"PowerUserPolicy"}}
+	bare := &Identity{Name: "bare_user"}
+
+	for _, action := range iamSelfServiceActionNames {
+		for _, identity := range []*Identity{powerUser, bare} {
+			for _, target := range []string{"", identity.Name, "victim"} {
+				assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), identity, action, target),
+					"%s by %s on %q", action, identity.Name, target)
+			}
+		}
+	}
+}
+
+// The grant is authorized against the target user's ARN, the caller's own when
+// UserName is empty, so a policy can confine it to the caller's own keys.
+func TestAuthorizeIamActionSelfServiceGrantIsScopedByResource(t *testing.T) {
+	for _, resource := range []string{"arn:aws:iam:::user/key_owner", "arn:aws:iam:::user/${aws:username}"} {
+		t.Run(resource, func(t *testing.T) {
+			iam := newIamAuthzTestIam(t)
+			iam.selfServiceRequiresGrant = true
+			require.NoError(t, iam.PutPolicy("OwnKeysPolicy", ownKeysPolicy(resource)))
+			owner := &Identity{Name: "key_owner", PolicyNames: []string{"OwnKeysPolicy"}}
+
+			for _, action := range iamSelfServiceActionNames {
+				assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, ""), "%s on itself, implicit", action)
+				assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, "key_owner"), "%s on itself", action)
+				assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, "victim"), "%s on another user", action)
+			}
+			assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), owner, "CreateUser", "victim"))
+		})
+	}
+}
+
+// The setting changes nothing for the admin, nor for an iam:* grant.
+func TestAuthorizeIamActionSelfServiceRequiresGrantKeepsAdmin(t *testing.T) {
+	iam := newIamAuthzTestIam(t)
+	iam.selfServiceRequiresGrant = true
+	admin := &Identity{Name: "admin", Actions: []Action{s3_constants.ACTION_ADMIN}}
+	operator := &Identity{Name: "iam_operator", PolicyNames: []string{"IamAdminPolicy"}}
+
+	for _, action := range iamSelfServiceActionNames {
+		for _, identity := range []*Identity{admin, operator} {
+			for _, target := range []string{"", identity.Name, "victim"} {
+				assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), identity, action, target),
+					"%s by %s on %q", action, identity.Name, target)
+			}
+		}
+	}
+}
+
+// Off by default: self-service stays allowed without a grant.
+func TestAuthorizeIamActionSelfServiceDefaultUnchanged(t *testing.T) {
+	iam := newIamAuthzTestIam(t)
+	powerUser := &Identity{Name: "power_user", PolicyNames: []string{"PowerUserPolicy"}}
+	for _, action := range iamSelfServiceActionNames {
+		assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), powerUser, action, ""), action)
+		assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), powerUser, action, "power_user"), action)
+	}
+}

weed/s3api/s3api_head_bucket_test.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"path"
-	"strings"
 	"testing"
 
 	"github.com/gorilla/mux"
@@ -56,19 +55,16 @@ func TestHeadBucketSeparatesLookupFailureFromMissingBucket(t *testing.T) {
 		name     string
 		filer    *fakeLookupFiler
 		wantCode int
-		wantBody string
 	}{
 		{
 			name:     "transient lookup failure",
 			filer:    &fakeLookupFiler{lookupErr: status.Error(codes.Internal, "filer store unavailable")},
 			wantCode: http.StatusInternalServerError,
-			wantBody: "<Code>InternalError</Code>",
 		},
 		{
 			name:     "missing bucket",
 			filer:    &fakeLookupFiler{},
 			wantCode: http.StatusNotFound,
-			wantBody: "<Code>NoSuchBucket</Code>",
 		},
 		{
 			name:     "existing bucket",
@@ -88,8 +84,10 @@ func TestHeadBucketSeparatesLookupFailureFromMissingBucket(t *testing.T) {
 			if rr.Code != tc.wantCode {
 				t.Fatalf("status = %d, want %d: %s", rr.Code, tc.wantCode, rr.Body.String())
 			}
-			if tc.wantBody != "" && !strings.Contains(rr.Body.String(), tc.wantBody) {
-				t.Fatalf("body = %s, want %s", rr.Body.String(), tc.wantBody)
+			// HEAD returns the status without an XML error body, including for
+			// lookup failures and missing buckets.
+			if rr.Body.Len() != 0 {
+				t.Fatalf("HEAD response must have an empty body, got %q", rr.Body.String())
 			}
 		})
 	}

weed/s3api/s3api_iceberg_credentials.go

@@ -2,12 +2,15 @@ package s3api
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/iam/sts"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 )
 
 // icebergVendedCredentialsProvider names the origin recorded on sessions the
@@ -34,6 +37,11 @@ type VendedCredentials struct {
 // has already authenticated and authorized. Returns nil when no role is
 // configured, which leaves the catalog telling the client to keep using its
 // own credentials.
+//
+// The access key the caller authenticated with (from ctx) is recorded in the
+// session: a short hash of it ends the session name, so the audit log's
+// requester_arn tells apart the holders of several keys on one identity, and
+// the key id itself is carried in the session token.
 func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) {
 	if s3a.icebergCredentialRole == "" {
 		return nil, nil
@@ -55,12 +63,14 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc
 		return nil, err
 	}
 
+	accessKey := s3_constants.AccessKeyIdFromContext(ctx)
 	request := &sts.AssumeRoleForPrincipalRequest{
-		RoleArn:         s3a.icebergCredentialRole,
-		Principal:       principal,
-		RoleSessionName: icebergSessionName(principal, bucket, prefix),
-		ProviderName:    icebergVendedCredentialsProvider,
-		Policy:          &policy,
+		RoleArn:           s3a.icebergCredentialRole,
+		Principal:         principal,
+		RoleSessionName:   icebergSessionName(principal, bucket, prefix, accessKey),
+		ProviderName:      icebergVendedCredentialsProvider,
+		Policy:            &policy,
+		SourceAccessKeyId: accessKey,
 	}
 	if s3a.icebergCredentialDuration > 0 {
 		duration := s3a.icebergCredentialDuration
@@ -162,9 +172,25 @@ func tablePrefixSessionPolicy(bucket, prefix string, readOnly bool) (string, err
 	return string(encoded), nil
 }
 
-// icebergSessionName builds a session name that identifies the caller and the
-// table in audit logs, within the 64-character AWS limit.
-func icebergSessionName(principal, bucket, prefix string) string {
+// icebergSessionKeyTagLength is the hex length of the access key hash that ends
+// a vended session's name: 48 bits, so two keys of one identity collide with
+// odds of about 2^-48.
+const icebergSessionKeyTagLength = 12
+
+// icebergSessionKeyTag is "-k" and the first icebergSessionKeyTagLength hex
+// digits of the SHA-256 of the access key id, or "" without a key.
+func icebergSessionKeyTag(accessKey string) string {
+	if accessKey == "" {
+		return ""
+	}
+	sum := sha256.Sum256([]byte(accessKey))
+	return "-k" + hex.EncodeToString(sum[:])[:icebergSessionKeyTagLength]
+}
+
+// icebergSessionName builds a session name that identifies the caller, the
+// table and the caller's access key in audit logs, within the 64-character AWS
+// limit. The key tag is never cut: the table part gives way to it.
+func icebergSessionName(principal, bucket, prefix, accessKey string) string {
 	name := fmt.Sprintf("iceberg-%s-%s", principal, strings.ReplaceAll(strings.Trim(prefix, "/"), "/", "-"))
 	if strings.Trim(prefix, "/") == "" {
 		name = fmt.Sprintf("iceberg-%s-%s", principal, bucket)
@@ -177,8 +203,9 @@ func icebergSessionName(principal, bucket, prefix string) string {
 			return '-'
 		}
 	}, name)
-	if len(name) > 64 {
-		name = name[:64]
+	tag := icebergSessionKeyTag(accessKey)
+	if limit := 64 - len(tag); len(name) > limit {
+		name = name[:limit]
 	}
-	return name
+	return name + tag
 }

weed/s3api/s3api_iceberg_credentials_test.go

@@ -114,7 +114,7 @@ func TestTablePrefixSessionPolicyLeavesBucketLocationUnconditioned(t *testing.T)
 }
 
 func TestIcebergSessionNameIsBounded(t *testing.T) {
-	name := icebergSessionName(strings.Repeat("principal", 10), "warehouse", "ns/table")
+	name := icebergSessionName(strings.Repeat("principal", 10), "warehouse", "ns/table", "")
 	if len(name) > 64 {
 		t.Errorf("session name is %d chars, want at most 64", len(name))
 	}