From 8fdcf69eb0ed733c2738168c745ebb2df7b44b69 Mon Sep 17 00:00:00 2001 From: Javier Garcia <93437997+IamYipi@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:11:19 +0200 Subject: [PATCH] s3api: report the stored checksum in GetObjectAttributes (#11529) GetObjectAttributes accepted the Checksum attribute but never filled it in, as its comment said SeaweedFS did not store S3 checksums. PutObject and CompleteMultipartUpload store them now, and HeadObject returns them. Fill in Checksum from the same entry fields, with the ChecksumType and ChecksumCRC64NVME members the response did not have. Also run ceph/s3-tests' test_get_checksum_object_attributes in CI. --- .github/workflows/s3tests.yml | 2 + .../s3api/s3api_object_handlers_attributes.go | 31 +++++-- .../s3api_object_handlers_attributes_test.go | 88 +++++++++++++++++++ 3 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 weed/s3api/s3api_object_handlers_attributes_test.go diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index b40f8d3ea..35ab1c177 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -312,6 +312,7 @@ jobs: s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_checksum_object_attributes \ s3tests/functional/test_s3.py::test_bucket_head \ s3tests/functional/test_s3.py::test_bucket_head_notexist \ s3tests/functional/test_s3.py::test_object_raw_authenticated \ @@ -1173,6 +1174,7 @@ jobs: s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \ s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \ s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \ + s3tests/functional/test_s3.py::test_get_checksum_object_attributes \ s3tests/functional/test_s3.py::test_bucket_head \ s3tests/functional/test_s3.py::test_bucket_head_notexist \ s3tests/functional/test_s3.py::test_object_raw_authenticated \ diff --git a/weed/s3api/s3api_object_handlers_attributes.go b/weed/s3api/s3api_object_handlers_attributes.go index 4f372e256..901409a26 100644 --- a/weed/s3api/s3api_object_handlers_attributes.go +++ b/weed/s3api/s3api_object_handlers_attributes.go @@ -29,10 +29,8 @@ type GetObjectAttributesResponse struct { // ObjectAttributesChecksum holds checksum info for GetObjectAttributes. type ObjectAttributesChecksum struct { - ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` - ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` - ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` - ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` + ChecksumResult + ChecksumType string `xml:"ChecksumType,omitempty"` } // ObjectAttributesParts holds parts info for GetObjectAttributes. @@ -77,6 +75,24 @@ func validateObjectAttributes(attrs map[string]struct{}) bool { return true } +// objectAttributesChecksum returns the additional checksum that PutObject or +// CompleteMultipartUpload stored with the object, or nil if it has none. +func objectAttributesChecksum(entry *filer_pb.Entry) *ObjectAttributesChecksum { + if entry == nil || entry.Extended == nil { + return nil + } + value := string(entry.Extended[s3_constants.ExtChecksumValue]) + if value == "" { + return nil + } + checksum := &ObjectAttributesChecksum{ChecksumType: string(entry.Extended[s3_constants.ExtChecksumType])} + checksum.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), value) + if checksum.ChecksumResult == (ChecksumResult{}) { + return nil + } + return checksum +} + func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("GetObjectAttributesHandler %s %s", bucket, object) @@ -245,10 +261,9 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt resp.StorageClass = storageClass } - // Checksum: accepted in validation so clients don't get a 400, but SeaweedFS - // does not yet store S3 checksums (CRC32, CRC32C, SHA1, SHA256), so - // resp.Checksum is intentionally left nil. When checksum storage is added, - // populate resp.Checksum here. + if _, ok := requestedAttrs["Checksum"]; ok { + resp.Checksum = objectAttributesChecksum(entry) + } if _, ok := requestedAttrs["ObjectSize"]; ok { var size int64 diff --git a/weed/s3api/s3api_object_handlers_attributes_test.go b/weed/s3api/s3api_object_handlers_attributes_test.go new file mode 100644 index 000000000..6138c267e --- /dev/null +++ b/weed/s3api/s3api_object_handlers_attributes_test.go @@ -0,0 +1,88 @@ +package s3api + +import ( + "encoding/xml" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestObjectAttributesChecksum verifies that GetObjectAttributes reports the +// checksum that PutObject or CompleteMultipartUpload stored with the object +func TestObjectAttributesChecksum(t *testing.T) { + testCases := []struct { + name string + extended map[string][]byte + want *ObjectAttributesChecksum + }{ + { + name: "PutObject with SHA256", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumSHA256), + s3_constants.ExtChecksumValue: []byte("arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumSHA256: "arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0="}, + }, + }, + { + name: "multipart upload with a composite CRC32C", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C), + s3_constants.ExtChecksumValue: []byte("x3Y2bw==-3"), + s3_constants.ExtChecksumType: []byte("COMPOSITE"), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC32C: "x3Y2bw==-3"}, + ChecksumType: "COMPOSITE", + }, + }, + { + name: "multipart upload with a full object CRC64NVME", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME), + s3_constants.ExtChecksumValue: []byte("AAAAAAAAAAA="), + s3_constants.ExtChecksumType: []byte("FULL_OBJECT"), + }, + want: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "AAAAAAAAAAA="}, + ChecksumType: "FULL_OBJECT", + }, + }, + { + name: "no checksum", + extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")}, + }, + { + name: "unknown algorithm", + extended: map[string][]byte{ + s3_constants.ExtChecksumAlgorithm: []byte("X-Amz-Checksum-Md5"), + s3_constants.ExtChecksumValue: []byte("1B2M2Y8AsgTpgAmY7PhCfg=="), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, objectAttributesChecksum(&filer_pb.Entry{Extended: tc.extended})) + }) + } + assert.Nil(t, objectAttributesChecksum(&filer_pb.Entry{})) +} + +// TestGetObjectAttributesChecksumXML verifies the Checksum element's layout +func TestGetObjectAttributesChecksumXML(t *testing.T) { + resp := GetObjectAttributesResponse{ + Checksum: &ObjectAttributesChecksum{ + ChecksumResult: ChecksumResult{ChecksumCRC32: "NhCmhg=="}, + ChecksumType: "FULL_OBJECT", + }, + } + out, err := xml.Marshal(resp) + require.NoError(t, err) + assert.Equal(t, "NhCmhg=="+ + "FULL_OBJECT", string(out)) +}