seaweedfs/weed/s3api/table_bucket_verify_object_test.go

package s3api

import (
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/gorilla/mux"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/stretchr/testify/assert"
)

const verifyTestBucket = "tables"

// verifyTestNow is the server clock the tests pin; keys are dated around it.
const verifyTestNow = 1790000000000

func newVerifyTestServer(t *testing.T) *S3ApiServer {
	t.Helper()
	saved := volumeVerifyNow
	volumeVerifyNow = func() time.Time { return time.UnixMilli(verifyTestNow) }
	savedLookup := volumeVerifyLookup
	volumeVerifyLookup = func(*S3ApiServer, string, string) (*filer_pb.Entry, error) { return nil, filer_pb.ErrNotFound }
	t.Cleanup(func() { volumeVerifyNow, volumeVerifyLookup = saved, savedLookup })
	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
	s3a.bucketRegistry = NewBucketRegistry(s3a)
	s3a.bucketRegistry.setMetadataCache(&BucketMetaData{Name: verifyTestBucket, IsTableBucket: true})
	s3a.bucketRegistry.setMetadataCache(&BucketMetaData{Name: "plain", IsTableBucket: false})
	return s3a
}

func TestTableBucketAcceptsExternalVolumeVerifyKey(t *testing.T) {
	s3a := newVerifyTestServer(t)
	for _, key := range []string{
		"verify_1790000000000_1081846483",
		"/verify_1790000000000_1",
		"verify_1790003300000_42", // 55 minutes later: inside the window
		"verify_1789996700000_42", // 55 minutes earlier
		// reads and deletes take a key of any date (PUT limits it)
		"verify_1789992000000_1",
		"verify_1790008000000_1",
	} {
		isVerify, err := s3a.validateTableBucketObjectPathOrVerify(verifyTestBucket, key)
		assert.True(t, isVerify, key)
		assert.NoError(t, err, key)
		// the handlers that keep the plain rule still refuse it
		assert.Error(t, s3a.validateTableBucketObjectPath(verifyTestBucket, key), key)
	}
	// the Iceberg layout is unchanged: the positive control
	isVerify, err := s3a.validateTableBucketObjectPathOrVerify(verifyTestBucket, "ns/t/data/part-00000.parquet")
	assert.False(t, isVerify)
	assert.NoError(t, err)
}

func TestTableBucketRefusesVerifyKeyLookalikes(t *testing.T) {
	s3a := newVerifyTestServer(t)
	for _, key := range []string{
		// the bucket root only
		"base/verify_1790000000000_1",
		"ns/verify_1790000000000_1",
		".uploads/verify_1790000000000_1",
		".uploads/abc/verify_1790000000000_1",
		// the exact shape only
		"verify_1790000000000_1.txt",
		"verify_1790000000000",
		"verify_179000000000_1",            // 12-digit timestamp
		"verify_1790000000000_12345678901", // 11-digit number
		"verify_abc_123",
		"verify_1790000000000_1/x",
		"xverify_1790000000000_1",
		// ordinary non-Iceberg keys
		"notes.txt",
		"ns/t/data/notes.txt",
	} {
		isVerify, err := s3a.validateTableBucketObjectPathOrVerify(verifyTestBucket, key)
		assert.False(t, isVerify, key)
		assert.Error(t, err, key)
	}
	// a bucket that is not a table bucket is unaffected
	isVerify, err := s3a.validateTableBucketObjectPathOrVerify("plain", "verify_1790000000000_1")
	assert.False(t, isVerify)
	assert.NoError(t, err)
}

func verifyPutRequest(key string) *http.Request {
	req := httptest.NewRequest(http.MethodPut, "/"+verifyTestBucket+"/"+key, strings.NewReader("x"))
	return mux.SetURLVars(req, map[string]string{"bucket": verifyTestBucket, "object": key})
}

// A PUT to the verification key cannot store data: a small declared body and
// nothing stored beside it.
func TestTableBucketVerifyKeyPutLimits(t *testing.T) {
	s3a := newVerifyTestServer(t)
	const key = "verify_1790000000000_1081846483"
	for name, mutate := range map[string]func(*http.Request){
		"too large":                    func(r *http.Request) { r.ContentLength = volumeVerifyMaxBytes + 1 },
		"content-language":             func(r *http.Request) { r.ContentLength = 0; r.Header.Set("Content-Language", "fr") },
		"content-encoding":             func(r *http.Request) { r.ContentLength = 0; r.Header.Set("Content-Encoding", "gzip") },
		"response-content-disposition": func(r *http.Request) { r.ContentLength = 0; r.Header.Set("Response-Content-Disposition", "attachment") },
		"extract query":                func(r *http.Request) { r.ContentLength = 0; r.URL.RawQuery = "extract=zip" },
		"extract header":               func(r *http.Request) { r.ContentLength = 0; r.Header.Set(s3_constants.SeaweedFSExtract, "zip") },
		"undeclared length":            func(r *http.Request) { r.ContentLength = -1 },
		"chunked":                      func(r *http.Request) { r.ContentLength = 0; r.TransferEncoding = []string{"chunked"} },
		"user metadata":                func(r *http.Request) { r.ContentLength = 0; r.Header.Set("X-Amz-Meta-Note", "x") },
		"tagging":                      func(r *http.Request) { r.ContentLength = 0; r.Header.Set("X-Amz-Tagging", "a=b") },
		"content-disposition":          func(r *http.Request) { r.ContentLength = 0; r.Header.Set("Content-Disposition", "attachment") },
		"website redirect":             func(r *http.Request) { r.ContentLength = 0; r.Header.Set("X-Amz-Website-Redirect-Location", "/x") },
	} {
		req := verifyPutRequest(key)
		mutate(req)
		assert.False(t, volumeVerifyPutAllowed(req, key), name)
		rec := httptest.NewRecorder()
		s3a.PutObjectHandler(rec, req)
		assert.Equal(t, http.StatusForbidden, rec.Code, name)
	}
	ok := verifyPutRequest(key)
	ok.ContentLength = 21
	ok.Header.Set("Content-Type", "application/octet-stream")
	ok.Header.Set("Content-Encoding", "aws-chunked")
	assert.True(t, volumeVerifyPutAllowed(ok, key), "the verification object itself")

	// a PUT is dated within the window of the server's clock
	for _, old := range []string{"verify_1789992000000_1", "verify_1790008000000_1"} {
		req := verifyPutRequest(old)
		req.ContentLength = 21
		assert.False(t, volumeVerifyPutAllowed(req, old), old)
		rec := httptest.NewRecorder()
		s3a.PutObjectHandler(rec, req)
		assert.Equal(t, http.StatusForbidden, rec.Code, old)
	}
}

// A namespace may carry a name of the key's shape; the key then names its
// directory and gets the plain layout rule, as does a key whose entry cannot
// be read.
func TestTableBucketVerifyKeyNamingDirectoryIsRefused(t *testing.T) {
	s3a := newVerifyTestServer(t)
	const key = "verify_1790000000000_7"
	for name, lookup := range map[string]func(*S3ApiServer, string, string) (*filer_pb.Entry, error){
		"namespace directory": func(*S3ApiServer, string, string) (*filer_pb.Entry, error) {
			return &filer_pb.Entry{Name: key, IsDirectory: true}, nil
		},
		"lookup failure": func(*S3ApiServer, string, string) (*filer_pb.Entry, error) {
			return nil, errors.New("filer unavailable")
		},
	} {
		volumeVerifyLookup = lookup
		isVerify, err := s3a.validateTableBucketObjectPathOrVerify(verifyTestBucket, key)
		assert.False(t, isVerify, name)
		assert.Error(t, err, name)
	}
	// the control: an existing verification object is a file
	volumeVerifyLookup = func(*S3ApiServer, string, string) (*filer_pb.Entry, error) {
		return &filer_pb.Entry{Name: key}, nil
	}
	isVerify, err := s3a.validateTableBucketObjectPathOrVerify(verifyTestBucket, key)
	assert.True(t, isVerify)
	assert.NoError(t, err)
}