mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-05 22:12:04 +02:00
s3api: copy the trailer checksum before reading the next trailer line (#11583)
* s3api: copy the trailer checksum before reading the next trailer line
parseChunkChecksum kept the checksum value as a sub-slice of the line
returned by bufio.Reader.ReadSlice, which is only valid until the next
read. When the trailer lines arrive in separate TCP segments, reading
x-amz-trailer-signature refills the buffer and overwrites the saved
value, so a correct upload fails with InvalidDigest ("The Content-Md5
you specified is not valid").
The AWS SDK for Java v2 (>= 2.30) on a Linux JDK sends the trailer that
way; about half of its signed streaming uploads failed.
Fixes #11582
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* s3api: reuse crc32 writer and trim comments in trailer split test
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
2 files changed
+60
-1
No files matched your search
@@ -0,0 +1,59 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// segmentedReader returns one segment per Read call.
|
||||
type segmentedReader struct{ segments [][]byte }
|
||||
|
||||
func (s *segmentedReader) Read(p []byte) (int, error) {
|
||||
if len(s.segments) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, s.segments[0])
|
||||
s.segments[0] = s.segments[0][n:]
|
||||
if len(s.segments[0]) == 0 {
|
||||
s.segments = s.segments[1:]
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestTrailerChecksumSurvivesSplitTrailerLines(t *testing.T) {
|
||||
payload := []byte("hello, trailer\n")
|
||||
crcWriter := crc32.NewIEEE()
|
||||
crcWriter.Write(payload)
|
||||
checksum := base64.StdEncoding.EncodeToString(crcWriter.Sum(nil))
|
||||
sig := "0000000000000000000000000000000000000000000000000000000000000000"
|
||||
|
||||
segments := [][]byte{
|
||||
[]byte("f;chunk-signature=" + sig + "\r\n"),
|
||||
append(payload, "\r\n"...),
|
||||
[]byte("0;chunk-signature=" + sig + "\r\n"),
|
||||
[]byte("x-amz-checksum-crc32:" + checksum),
|
||||
[]byte("\r\n"),
|
||||
[]byte("x-amz-trailer-signature:" + sig + "\r\n\r\n"),
|
||||
}
|
||||
|
||||
cr := &s3ChunkedReader{
|
||||
reader: bufio.NewReader(&segmentedReader{segments: segments}),
|
||||
chunkSHA256Writer: sha256.New(),
|
||||
checkSumAlgorithm: ChecksumAlgorithmCRC32.String(),
|
||||
checkSumWriter: getCheckSumWriter(ChecksumAlgorithmCRC32),
|
||||
state: readChunkHeader,
|
||||
hasTrailer: true,
|
||||
}
|
||||
|
||||
got, err := io.ReadAll(cr)
|
||||
if err != nil {
|
||||
t.Fatalf("read failed: %v", err)
|
||||
}
|
||||
if string(got) != string(payload) {
|
||||
t.Fatalf("payload = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user