diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go index 7664407ebc..9641195ea0 100644 --- a/weed/remote_storage/s3/s3_storage_client.go +++ b/weed/remote_storage/s3/s3_storage_client.go @@ -168,6 +168,27 @@ func S3CompatibleEndpoint(conf *remote_pb.RemoteConf) (string, bool) { return opt.endpoint, true } +// awsCredentialSourceConfigured reports whether the environment names a source +// for the AWS SDK default credential chain, such as static keys, a profile, +// IRSA web identity or the ECS/EKS container credentials endpoint. +func awsCredentialSourceConfigured() bool { + for _, name := range []string{ + "AWS_ACCESS_KEY_ID", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + } { + if os.Getenv(name) != "" { + return true + } + } + return false +} + // MakeWithHTTPClient builds the client for any S3-SDK-backed remote storage // type using the supplied *http.Client (or the AWS SDK default when nil). // Callers that need to pin the dial path against DNS rebinding can pass a @@ -191,12 +212,32 @@ func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (re } if opt.accessKey != "" && opt.secretKey != "" { config.Credentials = credentials.NewStaticCredentials(opt.accessKey, opt.secretKey, "") - } else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" { - // Explicitly disable signing for public buckets. + } else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" && !awsCredentialSourceConfigured() { + // Explicitly disable signing for public buckets. When the environment + // names an AWS credential source, leave Credentials nil so the SDK + // default chain (env, web identity, container, shared profile) is used. config.Credentials = credentials.AnonymousCredentials + } else if config.Credentials == nil && config.HTTPClient != nil { + // Remote credential providers (container endpoints, IMDS, web + // identity) fetch through this HTTP client; the guarded one is for the + // S3 endpoint and blocks the link-local credential endpoints, so the + // default chain resolves on an unguarded client. Shared config stays + // enabled so web identity and profiles resolve here too, and the + // region carries over so STS can sign assume-role requests. + credSess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{Region: config.Region, HTTPClient: &http.Client{}}, + SharedConfigState: session.SharedConfigEnable, + }) + if err != nil { + return nil, fmt.Errorf("create %s credential session: %w", opt.name, err) + } + config.Credentials = credSess.Config.Credentials } - sess, err := session.NewSession(config) + sess, err := session.NewSessionWithOptions(session.Options{ + Config: *config, + SharedConfigState: session.SharedConfigEnable, + }) if err != nil { return nil, fmt.Errorf("create %s session: %w", opt.name, err) } diff --git a/weed/remote_storage/s3/s3_storage_client_test.go b/weed/remote_storage/s3/s3_storage_client_test.go index 568e4a6333..dc404d5b18 100644 --- a/weed/remote_storage/s3/s3_storage_client_test.go +++ b/weed/remote_storage/s3/s3_storage_client_test.go @@ -21,6 +21,7 @@ import ( ) func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) { + clearAWSCredentialEnv(t) maker := s3RemoteStorageMaker{} conf := &remote_pb.RemoteConf{ Type: "s3", @@ -40,6 +41,74 @@ func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) { require.Same(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) } +func clearAWSCredentialEnv(t *testing.T) { + t.Helper() + for _, name := range []string{ + "AWS_ACCESS_KEY_ID", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + } { + t.Setenv(name, "") + } +} + +func TestS3MakeUsesDefaultCredentialChainWhenKeysAreEmptyAndSourceIsConfigured(t *testing.T) { + for name, value := range map[string]string{ + "AWS_ACCESS_KEY_ID": "env-access", + "AWS_PROFILE": "some-profile", + "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token", + "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:1/creds", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/creds", + } { + t.Run(name, func(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv(name, value) + t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/test") + + remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) + }) + } +} + +// A caller that pins a custom http.Client still gets the default chain, so +// remote providers resolve on an unguarded client while S3 requests use the +// pinned one. +func TestS3MakeResolvesEnvCredentialsWithCustomHTTPClient(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv("AWS_ACCESS_KEY_ID", "env-access") + t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret") + + remoteClient, err := MakeWithHTTPClient(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }, &http.Client{}) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) + value, err := s3Client.Config.Credentials.Get() + require.NoError(t, err) + require.Equal(t, "env-access", value.AccessKeyID) +} + func TestS3MakeUsesStaticCredentialsWhenKeysAreProvided(t *testing.T) { maker := s3RemoteStorageMaker{} conf := &remote_pb.RemoteConf{ @@ -396,3 +465,24 @@ func TestIsNoSuchKey(t *testing.T) { require.True(t, isNoSuchKey(fmt.Errorf("download: %w", awserr.New(awss3.ErrCodeNoSuchKey, "missing", nil)))) require.False(t, isNoSuchKey(awserr.New(awss3.ErrCodeNoSuchBucket, "missing", nil))) } + +func TestS3MakeResolvesEnvCredentialsWhenAnonymousDefault(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv("AWS_ACCESS_KEY_ID", "env-access") + t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret") + + remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + value, err := s3Client.Config.Credentials.Get() + require.NoError(t, err) + require.Equal(t, "env-access", value.AccessKeyID) + require.Equal(t, "env-secret", value.SecretAccessKey) +}