s3api: copy the trailer checksum before reading the next trailer line (#11583)

* s3api: copy the trailer checksum before reading the next trailer line

parseChunkChecksum kept the checksum value as a sub-slice of the line
returned by bufio.Reader.ReadSlice, which is only valid until the next
read. When the trailer lines arrive in separate TCP segments, reading
x-amz-trailer-signature refills the buffer and overwrites the saved
value, so a correct upload fails with InvalidDigest ("The Content-Md5
you specified is not valid").

The AWS SDK for Java v2 (>= 2.30) on a Linux JDK sends the trailer that
way; about half of its signed streaming uploads failed.

Fixes #11582

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3api: reuse crc32 writer and trim comments in trailer split test

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-10-04 01:14:15 +08:00
1 parent f4ef37e752
commit 39bc9cd0ef
2 files changed
+60 -1

No files matched your search

@@ -0,0 +1,59 @@
package s3api
import (
"bufio"
"crypto/sha256"
"encoding/base64"
"hash/crc32"
"io"
"testing"
)
// segmentedReader returns one segment per Read call.
type segmentedReader struct{ segments [][]byte }
func (s *segmentedReader) Read(p []byte) (int, error) {
if len(s.segments) == 0 {
return 0, io.EOF
}
n := copy(p, s.segments[0])
s.segments[0] = s.segments[0][n:]
if len(s.segments[0]) == 0 {
s.segments = s.segments[1:]
}
return n, nil
}
func TestTrailerChecksumSurvivesSplitTrailerLines(t *testing.T) {
payload := []byte("hello, trailer\n")
crcWriter := crc32.NewIEEE()
crcWriter.Write(payload)
checksum := base64.StdEncoding.EncodeToString(crcWriter.Sum(nil))
sig := "0000000000000000000000000000000000000000000000000000000000000000"
segments := [][]byte{
[]byte("f;chunk-signature=" + sig + "\r\n"),
append(payload, "\r\n"...),
[]byte("0;chunk-signature=" + sig + "\r\n"),
[]byte("x-amz-checksum-crc32:" + checksum),
[]byte("\r\n"),
[]byte("x-amz-trailer-signature:" + sig + "\r\n\r\n"),
}
cr := &s3ChunkedReader{
reader: bufio.NewReader(&segmentedReader{segments: segments}),
chunkSHA256Writer: sha256.New(),
checkSumAlgorithm: ChecksumAlgorithmCRC32.String(),
checkSumWriter: getCheckSumWriter(ChecksumAlgorithmCRC32),
state: readChunkHeader,
hasTrailer: true,
}
got, err := io.ReadAll(cr)
if err != nil {
t.Fatalf("read failed: %v", err)
}
if string(got) != string(payload) {
t.Fatalf("payload = %q, want %q", got, payload)
}
}