From eb744e71bdf5e739a7551b4919dac08d95f854e1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 6 Oct 2026 08:51:31 -0700 Subject: [PATCH] s3api: verify x-amz-content-sha256 against the streamed body A PUT carrying a hex or base64 payload hash now streams through a verifier that reports a mismatch once the stream is exhausted, instead of storing an object that does not match its declared hash. The error is deferred so intermediate reads that drop (n>0, err) results cannot silently swallow it. Sentinel values (unsigned/streaming payloads) remain exempt and malformed values fail fast. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/s3api_object_handlers_put.go | 8 +- weed/s3api/s3api_put_object_helper.go | 78 ++++++++++++- weed/s3api/s3api_put_object_helper_test.go | 123 +++++++++++++++++++++ weed/s3api/s3err/s3api_errors.go | 1 + 4 files changed, 208 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index e92c17107f..c2ca1a8e16 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -202,7 +202,11 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) dirContent, readErr = io.ReadAll(dataReader) if readErr != nil { glog.Errorf("PutObjectHandler: failed to read directory marker content %s/%s: %v", bucket, object, readErr) - s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + if strings.Contains(readErr.Error(), s3err.ErrMsgContentSha256Mismatch) { + s3err.WriteErrorResponse(w, r, s3err.ErrContentSHA256Mismatch) + } else { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + } return } } @@ -1536,6 +1540,8 @@ func mapChunkedUploadErrorToS3Error(reqCtx context.Context, err error) s3err.Err switch { case errors.Is(err, weed_server.ErrReadOnly): return s3err.ErrAccessDenied + case strings.Contains(err.Error(), s3err.ErrMsgContentSha256Mismatch): + return s3err.ErrContentSHA256Mismatch case strings.Contains(err.Error(), s3err.ErrMsgPayloadChecksumMismatch): return s3err.ErrInvalidDigest case errors.Is(err, operation.ErrTruncatedBody): diff --git a/weed/s3api/s3api_put_object_helper.go b/weed/s3api/s3api_put_object_helper.go index d020772e93..cc0d7b1c72 100644 --- a/weed/s3api/s3api_put_object_helper.go +++ b/weed/s3api/s3api_put_object_helper.go @@ -1,6 +1,12 @@ package s3api import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "hash" "io" "net/http" @@ -30,6 +36,76 @@ func getRequestDataReader(s3a *S3ApiServer, r *http.Request) (io.ReadCloser, s3e dataReader, s3ErrCode = s3a.iam.newChunkedReader(r) } } + if s3ErrCode != s3err.ErrNone { + return nil, s3ErrCode + } - return dataReader, s3ErrCode + expected, valid := expectedContentSha256(r) + if !valid { + return nil, s3err.ErrContentSHA256Mismatch + } + if expected != nil && dataReader != nil { + if r.ContentLength == 0 { + // Handlers may never read an empty body, so check it now. + emptySum := sha256.Sum256(nil) + if !bytes.Equal(expected, emptySum[:]) { + return nil, s3err.ErrContentSHA256Mismatch + } + return dataReader, s3err.ErrNone + } + dataReader = &contentSha256Verifier{reader: dataReader, hasher: sha256.New(), expected: expected} + } + return dataReader, s3err.ErrNone +} + +// expectedContentSha256 decodes the x-amz-content-sha256 header when it declares +// a payload hash — hex or base64 per SigV4 — so the body can be checked as it +// streams. Sentinel values (streaming, unsigned) carry no hash and are skipped; +// anything that decodes to neither returns valid=false. +func expectedContentSha256(r *http.Request) (expected []byte, valid bool) { + v := r.Header.Get("X-Amz-Content-Sha256") + if v == "" || v == unsignedPayload || v == streamingContentSHA256 || + v == streamingContentSHA256Trailer || v == streamingUnsignedPayload { + return nil, true + } + if decoded, err := hex.DecodeString(v); err == nil && len(decoded) == sha256.Size { + return decoded, true + } + if decoded, err := base64.StdEncoding.DecodeString(v); err == nil && len(decoded) == sha256.Size { + return decoded, true + } + return nil, false +} + +// contentSha256Verifier checks the decoded x-amz-content-sha256 once the stream +// is exhausted; handlers that never consume the body are unaffected. +type contentSha256Verifier struct { + reader io.ReadCloser + hasher hash.Hash + expected []byte + done bool + pending error +} + +func (v *contentSha256Verifier) Read(p []byte) (int, error) { + if v.pending != nil { + return 0, v.pending + } + n, err := v.reader.Read(p) + v.hasher.Write(p[:n]) + if err == io.EOF && !v.done { + v.done = true + if !bytes.Equal(v.hasher.Sum(nil), v.expected) { + v.pending = errors.New(s3err.ErrMsgContentSha256Mismatch) + err = nil + } + } + if n == 0 && v.pending != nil { + return 0, v.pending + } + return n, err +} + +func (v *contentSha256Verifier) Close() error { + return v.reader.Close() } diff --git a/weed/s3api/s3api_put_object_helper_test.go b/weed/s3api/s3api_put_object_helper_test.go index 8e6565783c..355a5984c1 100644 --- a/weed/s3api/s3api_put_object_helper_test.go +++ b/weed/s3api/s3api_put_object_helper_test.go @@ -1,7 +1,13 @@ package s3api import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "io" "net/http" + "net/http/httptest" "strings" "testing" @@ -183,3 +189,120 @@ func TestAuthTypeDetection(t *testing.T) { }) } } + +func sha256Hex(t *testing.T, b []byte) string { + t.Helper() + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func TestExpectedContentSha256(t *testing.T) { + good := sha256Hex(t, []byte("payload")) + + tests := []struct { + name string + header string + wantValid bool + wantNil bool + }{ + {"absent", "", true, true}, + {"unsigned", unsignedPayload, true, true}, + {"streaming signed", streamingContentSHA256, true, true}, + {"streaming trailer", streamingContentSHA256Trailer, true, true}, + {"streaming unsigned", streamingUnsignedPayload, true, true}, + {"hex", good, true, false}, + {"not hex", "nothex", false, true}, + {"short hex", "deadbeef", false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest("PUT", "/b/o", nil) + if tt.header != "" { + r.Header.Set("X-Amz-Content-Sha256", tt.header) + } + got, valid := expectedContentSha256(r) + if valid != tt.wantValid { + t.Fatalf("valid=%v want %v", valid, tt.wantValid) + } + if (got == nil) != tt.wantNil { + t.Fatalf("expected nil=%v, got %x", tt.wantNil, got) + } + }) + } + + t.Run("base64 decodes to sha256", func(t *testing.T) { + sum := sha256.Sum256([]byte("payload")) + r := httptest.NewRequest("PUT", "/b/o", nil) + r.Header.Set("X-Amz-Content-Sha256", base64.StdEncoding.EncodeToString(sum[:])) + got, valid := expectedContentSha256(r) + if !valid || !bytes.Equal(got, sum[:]) { + t.Fatalf("valid=%v got=%x", valid, got) + } + }) +} + +func newVerifier(body string, expectedHex string) *contentSha256Verifier { + expected, _ := hex.DecodeString(expectedHex) + return &contentSha256Verifier{ + reader: io.NopCloser(strings.NewReader(body)), + hasher: sha256.New(), + expected: expected, + } +} + +func TestContentSha256VerifierMatch(t *testing.T) { + v := newVerifier("hello", sha256Hex(t, []byte("hello"))) + got, err := io.ReadAll(v) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != "hello" { + t.Fatalf("got %q", got) + } +} + +func TestContentSha256VerifierMismatch(t *testing.T) { + v := newVerifier("hello", sha256Hex(t, []byte("other"))) + _, err := io.ReadAll(v) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("expected mismatch error, got %v", err) + } + if _, err := v.Read(make([]byte, 8)); err == nil { + t.Fatal("expected error to persist") + } +} + +// mimeDetect performs a single Read and ignores its error; the verifier must +// defer the mismatch so a dropped (n>0, err) result still surfaces later. +func TestContentSha256VerifierSurvivesDroppedError(t *testing.T) { + v := newVerifier("hello", sha256Hex(t, []byte("other"))) + buf := make([]byte, 512) + n, _ := v.Read(buf) + rest := io.MultiReader(bytes.NewReader(buf[:n]), v) + _, err := io.ReadAll(rest) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("expected deferred mismatch error, got %v", err) + } +} + +// A zero-length body never reaches the verifier, so the declared hash is +// compared against the empty-payload digest up front. +func TestGetRequestDataReaderEmptyBodyHash(t *testing.T) { + s3a := &S3ApiServer{ + iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)), + } + s3a.iam.isAuthEnabled = false + + mismatch := httptest.NewRequest("PUT", "/b/dir/", http.NoBody) + mismatch.Header.Set("X-Amz-Content-Sha256", sha256Hex(t, []byte("other"))) + if _, code := getRequestDataReader(s3a, mismatch); code != s3err.ErrContentSHA256Mismatch { + t.Fatalf("empty body with wrong hash: code=%v", code) + } + + emptySum := sha256.Sum256(nil) + match := httptest.NewRequest("PUT", "/b/dir/", http.NoBody) + match.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(emptySum[:])) + if _, code := getRequestDataReader(s3a, match); code != s3err.ErrNone { + t.Fatalf("empty body with empty hash: code=%v", code) + } +} diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index b2ce67dde6..ad5c0d3bef 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -180,6 +180,7 @@ const ( // Error message constants for checksum validation const ( + ErrMsgContentSha256Mismatch = "the provided 'x-amz-content-sha256' header does not match what was computed" ErrMsgPayloadChecksumMismatch = "payload checksum does not match" ErrMsgChunkSignatureMismatch = "chunk signature does not match" ErrMsgChecksumAlgorithmMismatch = "checksum algorithm mismatch"