seaweedfs/weed/s3api/s3api_object_handlers_conditional_read_test.go

package s3api

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)

// A precondition can only fail against an object that exists. GET/HEAD of a missing
// key must stay a missing-key answer even when If-Match or If-Unmodified-Since is sent.
func TestValidateConditionalHeadersForReadsMissingObject(t *testing.T) {
	s3a := &S3ApiServer{}

	existing := &filer_pb.Entry{
		Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
		Extended:   map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
	}
	deleteMarker := &filer_pb.Entry{
		Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
		Extended:   map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
	}
	future := time.Now().Add(24 * time.Hour).UTC().Format(http.TimeFormat)

	testCases := []struct {
		name   string
		header string
		value  string
		entry  *filer_pb.Entry
		want   s3err.ErrorCode
	}{
		{"if-match on missing object", s3_constants.IfMatch, "0000", nil, s3err.ErrNoSuchKey},
		{"if-match star on missing object", s3_constants.IfMatch, "*", nil, s3err.ErrNoSuchKey},
		{"if-unmodified-since on missing object", s3_constants.IfUnmodifiedSince, future, nil, s3err.ErrNoSuchKey},
		{"if-match on delete marker", s3_constants.IfMatch, "0000", deleteMarker, s3err.ErrNoSuchKey},
		{"if-none-match on missing object", s3_constants.IfNoneMatch, "*", nil, s3err.ErrNone},
		{"if-modified-since on missing object", s3_constants.IfModifiedSince, future, nil, s3err.ErrNone},
		{"if-match mismatch on existing object", s3_constants.IfMatch, "0000", existing, s3err.ErrPreconditionFailed},
		{"if-match hit on existing object", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", existing, s3err.ErrNone},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			r := httptest.NewRequest(http.MethodGet, "/bucket/object", nil)
			r.Header.Set(tc.header, tc.value)
			headers, errCode := parseConditionalHeaders(r)
			if errCode != s3err.ErrNone {
				t.Fatalf("parseConditionalHeaders: %v", errCode)
			}
			result := s3a.validateConditionalHeadersForReads(r, headers, tc.entry, "bucket", "object")
			if result.ErrorCode != tc.want {
				t.Errorf("got %v, want %v", result.ErrorCode, tc.want)
			}
		})
	}
}

func TestRemoteObjectETagAcrossCacheFill(t *testing.T) {
	s3a := &S3ApiServer{}
	entry := &filer_pb.Entry{
		Name:        "object",
		Attributes:  &filer_pb.FuseAttributes{FileSize: 2048, Mtime: 100},
		RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 2048, RemoteMtime: 100, RemoteETag: "remote-multipart-2"},
	}
	head := httptest.NewRecorder()
	s3a.setResponseHeaders(head, httptest.NewRequest(http.MethodHead, "/bucket/object", nil), entry, 2048)
	headETag := head.Header().Get("ETag")
	if headETag != `"remote-multipart-2"` {
		t.Errorf("cold ETag = %q", headETag)
	}
	entry.Chunks = []*filer_pb.FileChunk{
		{Offset: 0, Size: 1024, ETag: "1B2M2Y8AsgTpgAmY7PhCfg=="},
		{Offset: 1024, Size: 1024, ETag: "1B2M2Y8AsgTpgAmY7PhCfg=="},
	}
	entry.RemoteEntry.LastLocalSyncTsNs = 200
	if got := s3a.getObjectETag(entry); got != headETag {
		t.Errorf("cached ETag = %q, cold ETag = %q", got, headETag)
	}
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r := httptest.NewRequest(method, "/bucket/object", nil)
		r.Header.Set(s3_constants.IfMatch, headETag)
		r.Header.Set("Range", "bytes=1024-2047")
		headers, code := parseConditionalHeaders(r)
		if code != s3err.ErrNone {
			t.Fatal(code)
		}
		result := s3a.validateConditionalHeadersForReads(r, headers, entry, "bucket", "object")
		if result.ErrorCode != s3err.ErrNone {
			t.Errorf("%s after cache fill: %v", method, result.ErrorCode)
		}
	}
	entry.RemoteEntry.RemoteETag = `"remote-multipart-2"`
	if got := s3a.getObjectETag(entry); got != headETag {
		t.Errorf("quoted remote ETag = %q", got)
	}
	entry.Extended = map[string][]byte{s3_constants.ExtETagKey: []byte("local-etag")}
	if got := s3a.getObjectETag(entry); got != `"local-etag"` {
		t.Errorf("explicit ETag = %q", got)
	}
	entry.Extended = nil
	entry.RemoteEntry.LastLocalSyncTsNs = 0
	if got, want := s3a.getObjectETag(entry), s3a.calculateETagFromChunks(entry.Chunks); got != want {
		t.Errorf("unsynced local ETag = %q, want %q", got, want)
	}
}

// A metadata-only update such as utimens changes Mtime without touching the
// remote bytes, so the object must keep its remote ETag; otherwise a later
// cache fill restores it and breaks clients holding the interim value.
func TestRemoteObjectETagMetadataOnlyTouch(t *testing.T) {
	s3a := &S3ApiServer{}
	for _, cached := range []bool{false, true} {
		entry := &filer_pb.Entry{
			Name:       "object",
			Attributes: &filer_pb.FuseAttributes{FileSize: 2048, Mtime: 100},
			RemoteEntry: &filer_pb.RemoteEntry{
				RemoteSize: 2048, RemoteMtime: 100, RemoteETag: "remote-multipart-2",
			},
		}
		if cached {
			entry.Chunks = []*filer_pb.FileChunk{{Size: 2048, ETag: "1B2M2Y8AsgTpgAmY7PhCfg=="}}
			entry.RemoteEntry.LastLocalSyncTsNs = 200
		}
		entry.Attributes.Mtime = 101
		entry.Attributes.MtimeNs = 500
		if got := s3a.getObjectETag(entry); got != `"remote-multipart-2"` {
			t.Errorf("cached=%v touched ETag = %q", cached, got)
		}
	}
}

// Extending or shrinking a cached remote file through setattr changes FileSize
// while leaving its chunks and sync stamp alone; the remote ETag no longer
// describes the local bytes and conditional reads must not answer 304 for it.
func TestRemoteObjectETagLocalResize(t *testing.T) {
	s3a := &S3ApiServer{}
	for _, size := range []uint64{0, 1024, 4096} {
		entry := &filer_pb.Entry{
			Name:       "object",
			Attributes: &filer_pb.FuseAttributes{FileSize: 2048, Mtime: 100},
			Chunks:     []*filer_pb.FileChunk{{Size: 2048, ETag: "1B2M2Y8AsgTpgAmY7PhCfg=="}},
			RemoteEntry: &filer_pb.RemoteEntry{
				RemoteSize: 2048, RemoteMtime: 100, RemoteETag: "remote-multipart-2",
				LastLocalSyncTsNs: 200,
			},
		}
		entry.Attributes.FileSize = size
		if got := s3a.getObjectETag(entry); got == `"remote-multipart-2"` {
			t.Errorf("resized to %d retained remote ETag", size)
		}
		r := httptest.NewRequest(http.MethodHead, "/bucket/object", nil)
		r.Header.Set(s3_constants.IfNoneMatch, `"remote-multipart-2"`)
		headers, code := parseConditionalHeaders(r)
		if code != s3err.ErrNone {
			t.Fatal(code)
		}
		if got := s3a.validateConditionalHeadersForReads(r, headers, entry, "bucket", "object"); got.ErrorCode != s3err.ErrNone {
			t.Errorf("resized to %d read: %v", size, got.ErrorCode)
		}
	}
}

func TestRemoteObjectETagCopyConditions(t *testing.T) {
	s3a := &S3ApiServer{}
	entry := &filer_pb.Entry{
		Attributes:  &filer_pb.FuseAttributes{FileSize: 2048, Mtime: 100},
		RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 2048, RemoteMtime: 100, RemoteETag: "remote-multipart-2"},
	}
	for _, cached := range []bool{false, true} {
		if cached {
			entry.Chunks = []*filer_pb.FileChunk{{Size: 2048, ETag: "1B2M2Y8AsgTpgAmY7PhCfg=="}}
			entry.RemoteEntry.LastLocalSyncTsNs = 200
		}
		for _, condition := range []string{s3_constants.AmzCopySourceIfMatch, s3_constants.AmzCopySourceIfNoneMatch} {
			r := httptest.NewRequest(http.MethodPut, "/bucket/copy", nil)
			r.Header.Set(condition, s3a.getObjectETag(entry))
			want := s3err.ErrNone
			if condition == s3_constants.AmzCopySourceIfNoneMatch {
				want = s3err.ErrPreconditionFailed
			}
			if got := s3a.validateConditionalCopyHeaders(r, entry); got != want {
				t.Errorf("cached=%v %s: got %v, want %v", cached, condition, got, want)
			}
		}
	}
}

func TestRemoteObjectETagLocalChanges(t *testing.T) {
	s3a := &S3ApiServer{}
	for _, content := range [][]byte{nil, []byte("new local content")} {
		entry := &filer_pb.Entry{
			Attributes:  &filer_pb.FuseAttributes{FileSize: uint64(len(content)), Mtime: 101},
			Content:     content,
			RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 2048, RemoteMtime: 100, RemoteETag: "old-origin"},
		}
		if got, want := copyEntryETag(entry), strings.Trim(s3a.getObjectETag(entry), `"`); got != want {
			t.Errorf("local copy ETag = %q, read ETag = %q", got, want)
		}
		if got := s3a.getObjectETag(entry); got == `"old-origin"` {
			t.Errorf("local content %q retained remote ETag", content)
		}
		r := httptest.NewRequest(http.MethodHead, "/bucket/object", nil)
		r.Header.Set(s3_constants.IfNoneMatch, `"old-origin"`)
		headers, code := parseConditionalHeaders(r)
		if code != s3err.ErrNone {
			t.Fatal(code)
		}
		if got := s3a.validateConditionalHeadersForReads(r, headers, entry, "bucket", "object"); got.ErrorCode != s3err.ErrNone {
			t.Errorf("local content %q read: %v", content, got.ErrorCode)
		}
		r.Header.Set(s3_constants.AmzCopySourceIfMatch, `"old-origin"`)
		if got := s3a.validateConditionalCopyHeaders(r, entry); got != s3err.ErrPreconditionFailed {
			t.Errorf("local content %q copy: %v", content, got)
		}
	}
}

func TestRemoteObjectETagFallback(t *testing.T) {
	s3a := &S3ApiServer{}
	entry := &filer_pb.Entry{
		Attributes:  &filer_pb.FuseAttributes{Md5: []byte{0xab, 0xcd}},
		RemoteEntry: &filer_pb.RemoteEntry{},
	}
	if got := s3a.getObjectETag(entry); got != `"abcd"` {
		t.Errorf("missing remote ETag = %q", got)
	}
	entry.RemoteEntry.RemoteETag = "empty-origin"
	if got := s3a.getObjectETag(entry); got != `"empty-origin"` {
		t.Errorf("empty remote object ETag = %q", got)
	}
	entry.RemoteEntry = nil
	if got := s3a.getObjectETag(entry); got != `"abcd"` {
		t.Errorf("local object ETag = %q", got)
	}
}