From 2521c202aaf77a28452c28a0fe7ae4a20bef7519 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 16 Oct 2026 21:10:38 -0700 Subject: [PATCH] s3: accept signed streaming uploads when IAM is disabled (#11766) * s3: skip seed-signature verification in chunked reader when IAM is off When no credentials are configured there is nothing to verify streaming chunk signatures against. Leave the credential unset so the reader decodes the aws-chunked body and skips per-chunk verification, matching the unsigned-streaming path. This also fixes SigV4-signed unsigned-streaming uploads on open gateways, which previously failed in calculateSeedSignature. setupIam() now sets isAuthEnabled to match the credentials it installs, so the signature-verification tests still exercise the enabled path. * s3: decode signed streaming uploads when IAM is disabled AWS SDK v2 clients (Trino, Spark, modern aws-cli) send PutObject bodies as STREAMING-AWS4-HMAC-SHA256-PAYLOAD. On a gateway without S3 auth this was rejected with ErrAuthNotSetup, breaking writes to table buckets and regular buckets alike. The auth check already short-circuits when IAM is off, so route both streaming payload types through the chunked reader. * s3: cover signed trailer streaming uploads without IAM Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/chunked_reader_v4.go | 12 +++-- weed/s3api/chunked_reader_v4_test.go | 57 +++++++++++++++++++++- weed/s3api/s3api_put_object_helper.go | 22 ++------- weed/s3api/s3api_put_object_helper_test.go | 6 +-- 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index fb755a084..dc4a0b946 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -94,16 +94,20 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' or 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER' case streamingContentSHA256, streamingContentSHA256Trailer: glog.V(3).Infof("streaming content sha256 (with trailer: %v)", contentSha256Header == streamingContentSHA256Trailer) - credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req) - if errCode != s3err.ErrNone { - return nil, errCode + // With IAM disabled there is no credential to verify against; leave + // cred nil so chunk signatures are decoded but not verified. + if iam.isEnabled() { + credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req) + if errCode != s3err.ErrNone { + return nil, errCode + } } case streamingUnsignedPayload: glog.V(3).Infof("streaming unsigned payload") // The seed signature is a SigV4 concept; recompute it only for SigV4-signed // requests. JWT-bearer, presigned, and anonymous unsigned-streaming uploads // carry no header seed, and verifyV4Signature would fail parsing them. - if isRequestSignatureV4(req) { + if iam.isEnabled() && isRequestSignatureV4(req) { // We do not need to pass the seed signature to the Reader as each chunk is not signed, // but we do compute it to verify the request's signature. _, _, _, _, _, errCode = iam.calculateSeedSignature(req) diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go index e386129c0..e946dbb30 100644 --- a/weed/s3api/chunked_reader_v4_test.go +++ b/weed/s3api/chunked_reader_v4_test.go @@ -137,7 +137,7 @@ func setupIam() IdentityAccessManagement { hashCounters: map[string]*int32{}, identityAnonymous: nil, domain: "", - isAuthEnabled: false, + isAuthEnabled: true, } iam.identities = append(iam.identities, &Identity{ @@ -477,3 +477,58 @@ func TestSignedStreamingUploadInvalidSignature(t *testing.T) { assert.Error(t, err, "Expected error when reading chunk with invalid signature") assert.Contains(t, err.Error(), "chunk signature does not match", "Error should indicate chunk signature mismatch") } + +// TestSignedStreamingUploadWithoutIAM verifies that signed streaming uploads +// (STREAMING-AWS4-HMAC-SHA256-PAYLOAD) are decoded rather than rejected when no +// credentials are configured. Trino and other AWS SDK v2 clients always send +// this payload type, so open gateways must accept it. +func TestSignedStreamingUploadWithoutIAM(t *testing.T) { + iam := IdentityAccessManagement{ + identities: []*Identity{}, + accessKeyIdent: map[string]*Identity{}, + isAuthEnabled: false, + } + s3a := &S3ApiServer{iam: &iam} + + chunkData := strings.Repeat("a", 1024) + payload := fmt.Sprintf("400;chunk-signature=%064x\r\n%s\r\n", 1, chunkData) + + fmt.Sprintf("0;chunk-signature=%064x\r\n\r\n", 2) + + req, err := http.NewRequest(http.MethodPut, "http://localhost/test-bucket/test-object", + bytes.NewReader([]byte(payload))) + assert.NoError(t, err) + req.Header.Set("x-amz-content-sha256", streamingContentSHA256) + req.Header.Set("Content-Encoding", "aws-chunked") + req.Header.Set("x-amz-decoded-content-length", "1024") + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=x/20261016/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc") + + reader, errCode := getRequestDataReader(s3a, req) + assert.Equal(t, s3err.ErrNone, errCode) + assert.NotNil(t, reader) + + data, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, chunkData, string(data)) +} + +// TestTrailerStreamingUploadWithoutIAM verifies that the trailer variant +// (STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER) is also decoded when no +// credentials are configured, and that the checksum trailer is still honored. +func TestTrailerStreamingUploadWithoutIAM(t *testing.T) { + iam := IdentityAccessManagement{ + identities: []*Identity{}, + accessKeyIdent: map[string]*Identity{}, + isAuthEnabled: false, + } + s3a := &S3ApiServer{iam: &iam} + + req, expectedData := createTrailerStreamingRequest(t, true) + + reader, errCode := getRequestDataReader(s3a, req) + assert.Equal(t, s3err.ErrNone, errCode) + assert.NotNil(t, reader) + + data, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, expectedData, string(data)) +} diff --git a/weed/s3api/s3api_put_object_helper.go b/weed/s3api/s3api_put_object_helper.go index 12a861ddf..d6042aec5 100644 --- a/weed/s3api/s3api_put_object_helper.go +++ b/weed/s3api/s3api_put_object_helper.go @@ -14,27 +14,15 @@ import ( ) // getRequestDataReader returns the appropriate reader for the request body. -// When IAM is disabled, it still processes chunked transfer encoding for -// authTypeStreamingUnsigned to strip checksum headers and extract the actual data. -// This fixes issues where chunked data with checksums would be stored incorrectly -// when IAM is not enabled. +// Chunked streaming uploads are decoded regardless of whether IAM is enabled; +// with no credentials to verify against, signature checking is skipped inside +// newChunkedReader. func getRequestDataReader(s3a *S3ApiServer, r *http.Request) (io.ReadCloser, s3err.ErrorCode) { var s3ErrCode s3err.ErrorCode dataReader := r.Body rAuthType := getRequestAuthType(r) - if s3a.iam.isEnabled() { - if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingUnsigned { - dataReader, s3ErrCode = s3a.iam.newChunkedReader(r) - } - } else { - switch rAuthType { - case authTypeStreamingSigned: - s3ErrCode = s3err.ErrAuthNotSetup - case authTypeStreamingUnsigned: - // Even when IAM is disabled, we still need to handle chunked transfer encoding - // to strip checksum headers and process the data correctly - dataReader, s3ErrCode = s3a.iam.newChunkedReader(r) - } + if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingUnsigned { + dataReader, s3ErrCode = s3a.iam.newChunkedReader(r) } if s3ErrCode != s3err.ErrNone { return nil, s3ErrCode diff --git a/weed/s3api/s3api_put_object_helper_test.go b/weed/s3api/s3api_put_object_helper_test.go index 9328bb3db..292ce9dc7 100644 --- a/weed/s3api/s3api_put_object_helper_test.go +++ b/weed/s3api/s3api_put_object_helper_test.go @@ -41,9 +41,9 @@ func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) { { name: "StreamingSignedWithoutIAM", contentSha256: "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", - expectedError: s3err.ErrAuthNotSetup, - shouldProcess: false, - description: "Streaming signed requests should fail when IAM is disabled", + expectedError: s3err.ErrNone, + shouldProcess: true, + description: "Streaming signed requests are decoded when IAM is disabled; signatures are skipped since there is nothing to verify them against", }, { name: "StreamingUnsignedWithoutIAM",