seaweedfs/weed/s3api/filer_multipart_checksum_test.go

package s3api

import (
	"bytes"
	"encoding/base64"
	"hash/crc32"
	"testing"

	"github.com/minio/crc64nvme"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)

func makeCRC64NVMEPartEntry(data []byte) *filer_pb.Entry {
	sum := crc64nvme.New()
	sum.Write(data)
	return &filer_pb.Entry{
		Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(data))},
		Extended: map[string][]byte{
			s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC64NVME),
			s3_constants.ExtChecksumValue:     []byte(base64.StdEncoding.EncodeToString(sum.Sum(nil))),
		},
	}
}

func makeCRC32PartEntry(data []byte) *filer_pb.Entry {
	sum := crc32.NewIEEE()
	sum.Write(data)
	return &filer_pb.Entry{
		Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(data))},
		Extended: map[string][]byte{
			s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32),
			s3_constants.ExtChecksumValue:     []byte(base64.StdEncoding.EncodeToString(sum.Sum(nil))),
		},
	}
}

// Tests the FULL_OBJECT checksum of a multipart upload equals the CRC64NVME of the concatenated
// part data, with no composite "-N" suffix.
func TestComputeFullObjectChecksumCRC64NVME(t *testing.T) {
	parts := [][]byte{
		bytes.Repeat([]byte("a"), 5*1024*1024),
		bytes.Repeat([]byte("b"), 5*1024*1024),
		[]byte("tail part, smaller than the rest"),
	}

	partEntries := map[int][]*filer_pb.Entry{}
	var whole []byte
	completed := []int{}
	for i, data := range parts {
		partNumber := i + 1
		partEntries[partNumber] = []*filer_pb.Entry{makeCRC64NVMEPartEntry(data)}
		completed = append(completed, partNumber)
		whole = append(whole, data...)
	}

	got, err := computeFullObjectChecksum(s3_constants.AmzChecksumCRC64NVME, partEntries, completed)
	if err != nil {
		t.Fatalf("computeFullObjectChecksum: %v", err)
	}

	wholeSum := crc64nvme.New()
	wholeSum.Write(whole)
	expected := base64.StdEncoding.EncodeToString(wholeSum.Sum(nil))

	if got != expected {
		t.Fatalf("full object checksum = %q, want %q", got, expected)
	}
	if bytes.ContainsRune([]byte(got), '-') {
		t.Fatalf("full object checksum must not carry a -N suffix: %q", got)
	}
}

// Tests the FULL_OBJECT checksum of a multipart upload equals the CRC32 of the concatenated
// part data, with no composite "-N" suffix.
func TestComputeFullObjectChecksumCRC32(t *testing.T) {
	parts := [][]byte{
		bytes.Repeat([]byte("a"), 5*1024*1024),
		bytes.Repeat([]byte("b"), 5*1024*1024),
		[]byte("tail part, smaller than the rest"),
	}

	partEntries := map[int][]*filer_pb.Entry{}
	var whole []byte
	completed := []int{}
	for i, data := range parts {
		partNumber := i + 1
		partEntries[partNumber] = []*filer_pb.Entry{makeCRC32PartEntry(data)}
		completed = append(completed, partNumber)
		whole = append(whole, data...)
	}

	got, err := computeFullObjectChecksum(s3_constants.AmzChecksumCRC32, partEntries, completed)
	if err != nil {
		t.Fatalf("computeFullObjectChecksum: %v", err)
	}

	wholeSum := crc32.NewIEEE()
	wholeSum.Write(whole)
	expected := base64.StdEncoding.EncodeToString(wholeSum.Sum(nil))

	if got != expected {
		t.Fatalf("full object checksum = %q, want %q", got, expected)
	}
	if bytes.ContainsRune([]byte(got), '-') {
		t.Fatalf("full object checksum must not carry a -N suffix: %q", got)
	}
}

func TestResolveMultipartChecksumType(t *testing.T) {
	cases := []struct {
		name      string
		algo      ChecksumAlgorithm
		requested string
		want      string
		wantErr   bool
	}{
		{"crc64 default full", ChecksumAlgorithmCRC64NVMe, "", s3_constants.ChecksumTypeFullObject, false},
		{"crc64 explicit full", ChecksumAlgorithmCRC64NVMe, s3_constants.ChecksumTypeFullObject, s3_constants.ChecksumTypeFullObject, false},
		{"crc64 composite rejected", ChecksumAlgorithmCRC64NVMe, s3_constants.ChecksumTypeComposite, "", true},
		{"crc32 default composite", ChecksumAlgorithmCRC32, "", s3_constants.ChecksumTypeComposite, false},
		{"crc32 explicit composite", ChecksumAlgorithmCRC32, s3_constants.ChecksumTypeComposite, s3_constants.ChecksumTypeComposite, false},
		{"crc32 explicit full", ChecksumAlgorithmCRC32, s3_constants.ChecksumTypeFullObject, s3_constants.ChecksumTypeFullObject, false},
		{"crc32c explicit full", ChecksumAlgorithmCRC32C, s3_constants.ChecksumTypeFullObject, s3_constants.ChecksumTypeFullObject, false},
		{"sha256 default composite", ChecksumAlgorithmSHA256, "", s3_constants.ChecksumTypeComposite, false},
		{"sha256 full rejected", ChecksumAlgorithmSHA256, s3_constants.ChecksumTypeFullObject, "", true},
		{"invalid checksum type", ChecksumAlgorithmCRC64NVMe, "bogus", "", true},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got, err := resolveMultipartChecksumType(tc.algo, tc.requested)
			if tc.wantErr {
				if err == nil {
					t.Fatalf("expected error, got %q", got)
				}
				return
			}
			if err != nil {
				t.Fatalf("unexpected error: %v", err)
			}
			if got != tc.want {
				t.Fatalf("got %q, want %q", got, tc.want)
			}
		})
	}
}