mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 01:10:41 +01:00
test: add s3 integration tests for SQLite Cloud Backed Storage (#12243)
* test: add s3 integration tests for SQLite Cloud Backed Storage (#12235) SQLite CBS stores a database as named objects in a bucket and relies on conditional PUT — put-if-absent for new blocks and ETag compare-and-swap on the manifest — so concurrent clients cannot corrupt the database. The suite runs a weed mini cluster (conditions are evaluated under the object's write lock on the owner filer via the routed-write path) and covers: - the CBS storage primitives: create/put/fetch/list/delete/destroy - If-None-Match: * put-if-absent, success and 412 rejection - If-Match ETag compare-and-swap, including stale-ETag rejection and the re-read/retry loop CBS clients run on 412, on plain and versioned buckets - racing put-if-absent and manifest CAS writers: exactly one commit wins, every update survives - conditional CompleteMultipartUpload, sequential and raced - an end-to-end bcv-style upload, download-and-verify, and checkpoint A matching job is added to the S3 mutation regression workflow. * test/s3: drop issue references from the sqlite cbs suite comments * test/s3: abort every losing upload after the conditional multipart race * test/s3: assert the winning writer's body after the conditional write races * test/s3: spell out the get-returned etag in the cbs suite * test/s3: assert the completed multipart ETag the cbs suite feeds back The conditional-completion checks pass completeOut.ETag into If-Match; if the response carried no ETag the check would silently run unconditional. * test/s3: drop the unused generation parameter from the cbs block body
This commit is contained in:
Chris Lu authored and GitHub committed
1 parent
12dac68d66
commit
e80430db8e
3 files changed
+1205
No files matched your search
@@ -0,0 +1,896 @@
|
||||
package sqlite_cbs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// These tests exercise SeaweedFS the way SQLite Cloud Backed Storage (CBS)
|
||||
// drives a cloud bucket. CBS needs put/fetch/delete/list primitives and —
|
||||
// critically — conditional PUT semantics so two clients cannot clobber each
|
||||
// other's manifest update: put-if-absent (If-None-Match: *) for new objects
|
||||
// and compare-and-swap (If-Match: "<etag>") for manifest rewrites. A failed
|
||||
// precondition must return 412 and must not commit, even when conditional
|
||||
// writes to the same key race. See sqlite.org/cloudsqlite/doc/trunk/www/module.wiki.
|
||||
|
||||
func cbsTestContext() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 30*time.Second)
|
||||
}
|
||||
|
||||
func cbsBucketName(prefix string) string {
|
||||
return fmt.Sprintf("cbs-%s-%d", prefix, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func createCBSBucket(t *testing.T, client *s3.Client, bucket string) {
|
||||
t.Helper()
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func putCBSTestObject(t *testing.T, client *s3.Client, bucket, key, body string) *s3.PutObjectOutput {
|
||||
t.Helper()
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
out, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte(body)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.ETag, "PutObject must return the ETag CBS feeds back into If-Match")
|
||||
return out
|
||||
}
|
||||
|
||||
func getCBSTestObject(t *testing.T, client *s3.Client, bucket, key string) (etag string, body []byte) {
|
||||
t.Helper()
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
out, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer out.Body.Close()
|
||||
body, err = io.ReadAll(out.Body)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.ETag, "GetObject must return the ETag CBS feeds back into If-Match")
|
||||
return aws.ToString(out.ETag), body
|
||||
}
|
||||
|
||||
func isPreconditionFailed(err error) bool {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "PreconditionFailed" {
|
||||
return true
|
||||
}
|
||||
var respErr *awshttp.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusPreconditionFailed
|
||||
}
|
||||
|
||||
func isNotModified(err error) bool {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotModified" {
|
||||
return true
|
||||
}
|
||||
var respErr *awshttp.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotModified
|
||||
}
|
||||
|
||||
func isNoSuchKey(err error) bool {
|
||||
var noSuchKey *types.NoSuchKey
|
||||
if errors.As(err, &noSuchKey) {
|
||||
return true
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode() == "NoSuchKey"
|
||||
}
|
||||
|
||||
// TestCBSContainerAndObjectPrimitives covers the storage primitives a CBS
|
||||
// cloud module needs: create/destroy a container (bucket), put, fetch, list,
|
||||
// head and delete named objects.
|
||||
func TestCBSContainerAndObjectPrimitives(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("primitives")
|
||||
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
|
||||
_, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err, "container should be reachable right after creation")
|
||||
|
||||
// xPut: store named block objects under a database prefix.
|
||||
blocks := map[string]string{
|
||||
"mydb/manifest": "gen=1 blocks=block-000,block-001",
|
||||
"mydb/block-000": "page data for block 0",
|
||||
"mydb/block-001": "page data for block 1",
|
||||
}
|
||||
for key, body := range blocks {
|
||||
putCBSTestObject(t, client, bucket, key, body)
|
||||
}
|
||||
|
||||
// xFetch: read objects back by name.
|
||||
for key, want := range blocks {
|
||||
_, got := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, want, string(got), "fetched body for %s", key)
|
||||
}
|
||||
|
||||
// xList: enumerate the container's objects under the database prefix.
|
||||
listOut, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String("mydb/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
gotKeys := make([]string, 0, len(listOut.Contents))
|
||||
for _, obj := range listOut.Contents {
|
||||
gotKeys = append(gotKeys, aws.ToString(obj.Key))
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"mydb/manifest", "mydb/block-000", "mydb/block-001"}, gotKeys)
|
||||
|
||||
// A conditional fetch (xFetch carries an ETag) should answer 304 when the
|
||||
// cached copy is current, so a CBS daemon revalidating a cached block does
|
||||
// not re-download it.
|
||||
manifestETag, _ := getCBSTestObject(t, client, bucket, "mydb/manifest")
|
||||
getOut, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("mydb/manifest"),
|
||||
IfNoneMatch: aws.String(manifestETag),
|
||||
})
|
||||
if getOut != nil && getOut.Body != nil {
|
||||
getOut.Body.Close()
|
||||
}
|
||||
require.Error(t, err, "If-None-Match with the current ETag must not return the object again")
|
||||
assert.True(t, isNotModified(err), "expected 304 NotModified, got %v", err)
|
||||
|
||||
// xDelete: drop a named object.
|
||||
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("mydb/block-001"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("mydb/block-001"),
|
||||
})
|
||||
require.Error(t, err, "deleted block should be gone")
|
||||
|
||||
// xDestroy: remove the remaining objects and the container itself.
|
||||
for _, key := range []string{"mydb/manifest", "mydb/block-000"} {
|
||||
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, err = client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestCBSPutIfAbsent covers the put-if-absent half of CBS's conditional-PUT
|
||||
// requirement: If-None-Match: * must create the object when absent and fail
|
||||
// with 412 without committing when the name is taken.
|
||||
func TestCBSPutIfAbsent(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("if-absent")
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
key := "mydb/block-007"
|
||||
|
||||
first, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte("first")),
|
||||
})
|
||||
require.NoError(t, err, "put-if-absent on a new block name must succeed")
|
||||
require.NotNil(t, first.ETag)
|
||||
|
||||
second, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte("second")),
|
||||
})
|
||||
require.Error(t, err, "put-if-absent on a taken block name must fail")
|
||||
assert.True(t, isPreconditionFailed(err), "expected 412 PreconditionFailed, got %v", err)
|
||||
assert.Nil(t, second)
|
||||
|
||||
_, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, "first", string(body), "the rejected conditional PUT must not have committed")
|
||||
}
|
||||
|
||||
// TestCBSCompareAndSwap covers the compare-and-swap half: If-Match carries the
|
||||
// version identifier from a prior PUT/GET and the write only commits when the
|
||||
// stored object is still exactly that version.
|
||||
func TestCBSCompareAndSwap(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("cas")
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
key := "mydb/manifest"
|
||||
|
||||
v1 := putCBSTestObject(t, client, bucket, key, "generation=1")
|
||||
etag1 := aws.ToString(v1.ETag)
|
||||
|
||||
// If-Match with the version identifier just returned commits.
|
||||
v2, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: aws.String(etag1),
|
||||
Body: bytes.NewReader([]byte("generation=2")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etag2 := aws.ToString(v2.ETag)
|
||||
require.NotEqual(t, etag1, etag2)
|
||||
|
||||
// A retry of the same conditional write — the identifier is now stale —
|
||||
// must fail instead of clobbering generation 2.
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: aws.String(etag1),
|
||||
Body: bytes.NewReader([]byte("generation=1-replayed")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err), "stale If-Match must yield 412, got %v", err)
|
||||
|
||||
// The current identifier still works.
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: aws.String(etag2),
|
||||
Body: bytes.NewReader([]byte("generation=3")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, "generation=3", string(body))
|
||||
|
||||
// If-Match on a name that does not exist can never commit — there is no
|
||||
// version it could equal. SeaweedFS deliberately answers 404 NoSuchKey
|
||||
// here rather than 412 (see validateConditionalHeaders); CBS only needs
|
||||
// the write to fail, so accept either rejection.
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("mydb/no-such-block"),
|
||||
IfMatch: aws.String(etag2),
|
||||
Body: bytes.NewReader([]byte("orphan")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err) || isNoSuchKey(err), "If-Match on an absent key must be rejected, got %v", err)
|
||||
|
||||
// If-Match: * is "any version exists": commits over an existing object,
|
||||
// fails on an absent one (404 NoSuchKey here, same as above).
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte("generation=4")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("mydb/another-missing-block"),
|
||||
IfMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte("orphan")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err) || isNoSuchKey(err), "If-Match:* on an absent key must be rejected, got %v", err)
|
||||
|
||||
// If-None-Match with a concrete identifier fails only when it equals the
|
||||
// current version — the form a client uses for "put unless unchanged".
|
||||
currentETag, _ := getCBSTestObject(t, client, bucket, key)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String(currentETag),
|
||||
Body: bytes.NewReader([]byte("unchanged rewrite")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err), "If-None-Match with the current ETag must yield 412, got %v", err)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String(etag1),
|
||||
Body: bytes.NewReader([]byte("generation=5")),
|
||||
})
|
||||
require.NoError(t, err, "If-None-Match with an outdated ETag must commit")
|
||||
}
|
||||
|
||||
// TestCBSCompareAndSwapVersionedBucket runs the same CAS sequence on a
|
||||
// versioned bucket. Conditional writes must validate against the latest
|
||||
// version rather than a stale entry.
|
||||
func TestCBSCompareAndSwapVersionedBucket(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("cas-versioned")
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
|
||||
_, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucket),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
key := "mydb/manifest"
|
||||
v1 := putCBSTestObject(t, client, bucket, key, "generation=1")
|
||||
v2 := putCBSTestObject(t, client, bucket, key, "generation=2")
|
||||
|
||||
// The latest version's identifier commits; v1's is stale and must fail.
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: v1.ETag,
|
||||
Body: bytes.NewReader([]byte("stale rewrite")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err), "If-Match against a non-latest version must yield 412, got %v", err)
|
||||
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: v2.ETag,
|
||||
Body: bytes.NewReader([]byte("generation=3")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// The CAS also works fed from the ETag a GET returned, matching how CBS
|
||||
// re-reads the manifest before rewriting it.
|
||||
etag3, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, "generation=3", string(body))
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: aws.String(etag3),
|
||||
Body: bytes.NewReader([]byte("generation=4")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestCBSConcurrentPutIfAbsent races several CBS writers creating the same new
|
||||
// block name: exactly one put-if-absent may commit.
|
||||
func TestCBSConcurrentPutIfAbsent(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
bucket := cbsBucketName("race-absent")
|
||||
adminClient := cluster.newS3Client(t)
|
||||
createCBSBucket(t, adminClient, bucket)
|
||||
|
||||
const writers = 8
|
||||
key := "mydb/block-contended"
|
||||
|
||||
start := make(chan struct{})
|
||||
type putResult struct {
|
||||
writer int
|
||||
err error
|
||||
}
|
||||
results := make(chan putResult, writers)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < writers; i++ {
|
||||
wg.Add(1)
|
||||
// Separate clients stand in for separate CBS client processes.
|
||||
writerClient := cluster.newS3Client(t)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_, err := writerClient.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte(fmt.Sprintf("writer-%d", i))),
|
||||
})
|
||||
results <- putResult{i, err}
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
successes, preconditionFailures, winner := 0, 0, -1
|
||||
for res := range results {
|
||||
switch {
|
||||
case res.err == nil:
|
||||
successes++
|
||||
winner = res.writer
|
||||
case isPreconditionFailed(res.err):
|
||||
preconditionFailures++
|
||||
default:
|
||||
t.Fatalf("unexpected race error: %v", res.err)
|
||||
}
|
||||
}
|
||||
require.Equal(t, 1, successes, "exactly one put-if-absent may commit")
|
||||
require.Equal(t, writers-1, preconditionFailures, "every losing write must see 412")
|
||||
|
||||
// A losing write must not have committed over the winner's data.
|
||||
_, body := getCBSTestObject(t, adminClient, bucket, key)
|
||||
assert.Equal(t, fmt.Sprintf("writer-%d", winner), string(body))
|
||||
}
|
||||
|
||||
// cbsManifest mirrors the state a CBS daemon keeps in cloud storage: a single
|
||||
// object naming the current generation of database blocks, rewritten
|
||||
// transactionally at each checkpoint.
|
||||
type cbsManifest struct {
|
||||
Updates []string `json:"updates"`
|
||||
}
|
||||
|
||||
// TestCBSConcurrentManifestCAS is the core CBS correctness scenario: many
|
||||
// clients read-modify-write the same manifest object. Every writer retries its
|
||||
// read + If-Match PUT on 412, so all commits land and no update is silently
|
||||
// lost — which is exactly what keeps the database intact when two cloud
|
||||
// clients checkpoint at the same time.
|
||||
func TestCBSConcurrentManifestCAS(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
bucket := cbsBucketName("manifest-cas")
|
||||
adminClient := cluster.newS3Client(t)
|
||||
createCBSBucket(t, adminClient, bucket)
|
||||
|
||||
const writers = 8
|
||||
key := "mydb/manifest"
|
||||
|
||||
// Seed the manifest so every writer takes the read-then-CAS path.
|
||||
putCBSTestObject(t, adminClient, bucket, key, `{"updates":[]}`)
|
||||
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, writers)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < writers; i++ {
|
||||
wg.Add(1)
|
||||
writerTag := fmt.Sprintf("writer-%d", i)
|
||||
writerClient := cluster.newS3Client(t)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
errs <- casManifestUpdate(writerClient, bucket, key, writerTag)
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
|
||||
for err := range errs {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Every committed update must be present exactly once: a lost CAS would
|
||||
// drop a checkpoint's block list and corrupt the database.
|
||||
finalETag, body := getCBSTestObject(t, adminClient, bucket, key)
|
||||
require.NotEmpty(t, finalETag)
|
||||
var manifest cbsManifest
|
||||
require.NoError(t, json.Unmarshal(body, &manifest))
|
||||
assert.Len(t, manifest.Updates, writers)
|
||||
for i := 0; i < writers; i++ {
|
||||
assert.Contains(t, manifest.Updates, fmt.Sprintf("writer-%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
// casManifestUpdate performs one CBS-style manifest rewrite: fetch the current
|
||||
// manifest and its version identifier, append the writer's tag, and commit
|
||||
// with If-Match. A 412 means another client committed first — re-read and
|
||||
// retry, which is what a CBS client must do on a failed conditional PUT.
|
||||
func casManifestUpdate(client *s3.Client, bucket, key, tag string) error {
|
||||
for attempt := 0; attempt < 64; attempt++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
getOut, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("fetch manifest: %w", err)
|
||||
}
|
||||
raw, err := io.ReadAll(getOut.Body)
|
||||
getOut.Body.Close()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
var manifest cbsManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
manifest.Updates = append(manifest.Updates, tag)
|
||||
updated, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfMatch: getOut.ETag,
|
||||
Body: bytes.NewReader(updated),
|
||||
})
|
||||
cancel()
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case isPreconditionFailed(err):
|
||||
continue // another client committed first; re-read and retry
|
||||
default:
|
||||
return fmt.Errorf("commit manifest: %w", err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("writer %s never won the manifest CAS", tag)
|
||||
}
|
||||
|
||||
// TestCBSConditionalMultipartCommit covers conditional headers on
|
||||
// CompleteMultipartUpload. CBS daemons stage large uploads; the completion
|
||||
// must apply the same conditions atomically with the publish, or two racing
|
||||
// completions could both report success.
|
||||
func TestCBSConditionalMultipartCommit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("mp-cond")
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
key := "mydb/big-block"
|
||||
|
||||
// If-None-Match: * completes when the object is absent.
|
||||
uploadID, partETag := startMultipartUpload(t, client, bucket, key, "multipart-generation-1")
|
||||
completeOut, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
|
||||
{ETag: aws.String(partETag), PartNumber: aws.Int32(1)},
|
||||
}},
|
||||
IfNoneMatch: aws.String("*"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, completeOut.ETag, "CompleteMultipartUpload must return the ETag the If-Match checks feed back")
|
||||
|
||||
// A second conditional completion on the now-existing key must fail
|
||||
// instead of silently replacing the object.
|
||||
uploadID2, partETag2 := startMultipartUpload(t, client, bucket, key, "multipart-generation-2")
|
||||
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID2),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
|
||||
{ETag: aws.String(partETag2), PartNumber: aws.Int32(1)},
|
||||
}},
|
||||
IfNoneMatch: aws.String("*"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err), "expected 412 on conditional multipart completion, got %v", err)
|
||||
abortMultipartUpload(client, bucket, key, uploadID2)
|
||||
|
||||
// If-Match commits when fed the stored version identifier, and the stale
|
||||
// identifier from before must then fail.
|
||||
uploadID3, partETag3 := startMultipartUpload(t, client, bucket, key, "multipart-generation-3")
|
||||
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID3),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
|
||||
{ETag: aws.String(partETag3), PartNumber: aws.Int32(1)},
|
||||
}},
|
||||
IfMatch: completeOut.ETag,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
uploadID4, partETag4 := startMultipartUpload(t, client, bucket, key, "multipart-generation-4")
|
||||
_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID4),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
|
||||
{ETag: aws.String(partETag4), PartNumber: aws.Int32(1)},
|
||||
}},
|
||||
IfMatch: completeOut.ETag,
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err), "stale If-Match on multipart completion must yield 412, got %v", err)
|
||||
abortMultipartUpload(client, bucket, key, uploadID4)
|
||||
|
||||
_, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, "multipart-generation-3", string(body))
|
||||
}
|
||||
|
||||
// TestCBSConcurrentConditionalMultipartCommit races two completions of the
|
||||
// same key, each guarded by put-if-absent. The race the post-4.51 work fixed:
|
||||
// previously both completions could report success.
|
||||
func TestCBSConcurrentConditionalMultipartCommit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
bucket := cbsBucketName("mp-race")
|
||||
adminClient := cluster.newS3Client(t)
|
||||
createCBSBucket(t, adminClient, bucket)
|
||||
|
||||
const racers = 4
|
||||
key := "mydb/raced-block"
|
||||
|
||||
uploadIDs := make([]string, racers)
|
||||
partETags := make([]string, racers)
|
||||
for i := 0; i < racers; i++ {
|
||||
uploadIDs[i], partETags[i] = startMultipartUpload(t, adminClient, bucket, key, fmt.Sprintf("racer-%d", i))
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
type raceResult struct {
|
||||
racer int
|
||||
err error
|
||||
}
|
||||
results := make(chan raceResult, racers)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < racers; i++ {
|
||||
wg.Add(1)
|
||||
racerClient := cluster.newS3Client(t)
|
||||
uploadID, partETag := uploadIDs[i], partETags[i]
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_, err := racerClient.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{
|
||||
{ETag: aws.String(partETag), PartNumber: aws.Int32(1)},
|
||||
}},
|
||||
IfNoneMatch: aws.String("*"),
|
||||
})
|
||||
results <- raceResult{i, err}
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
successes, preconditionFailures, winner := 0, 0, -1
|
||||
for res := range results {
|
||||
switch {
|
||||
case res.err == nil:
|
||||
successes++
|
||||
winner = res.racer
|
||||
case isPreconditionFailed(res.err):
|
||||
preconditionFailures++
|
||||
default:
|
||||
t.Fatalf("unexpected race error: %v", res.err)
|
||||
}
|
||||
}
|
||||
require.Equal(t, 1, successes, "exactly one conditional multipart completion may commit")
|
||||
require.Equal(t, racers-1, preconditionFailures, "every losing completion must see 412")
|
||||
|
||||
// A losing completion must not have published over the winner's data.
|
||||
_, body := getCBSTestObject(t, adminClient, bucket, key)
|
||||
assert.Equal(t, fmt.Sprintf("racer-%d", winner), string(body))
|
||||
|
||||
for _, uploadID := range uploadIDs {
|
||||
abortMultipartUpload(adminClient, bucket, key, uploadID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCBSDatabaseUploadDownloadRoundtrip replays a bcv-style session end to
|
||||
// end: a client uploads immutable database blocks plus a manifest guarded by
|
||||
// put-if-absent, a second client lists and fetches the database back, then a
|
||||
// checkpoint writes new blocks and rewrites the manifest via CAS while a stale
|
||||
// writer's CAS is rejected.
|
||||
func TestCBSDatabaseUploadDownloadRoundtrip(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping sqlite cbs integration test in short mode")
|
||||
}
|
||||
cluster := startCBSMiniCluster(t)
|
||||
client := cluster.newS3Client(t)
|
||||
bucket := cbsBucketName("roundtrip")
|
||||
createCBSBucket(t, client, bucket)
|
||||
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
prefix := "shop.db/"
|
||||
manifestKey := prefix + "manifest"
|
||||
|
||||
const blockCount = 6
|
||||
blockBody := func(i int) []byte {
|
||||
return bytes.Repeat([]byte{byte('a' + i)}, 1024) // deterministic content per block
|
||||
}
|
||||
blockKey := func(i int) string {
|
||||
return fmt.Sprintf("%sblock-%06d", prefix, i)
|
||||
}
|
||||
|
||||
// Upload generation 1: immutable blocks with put-if-absent, then the
|
||||
// manifest guarded the same way so no existing database is clobbered.
|
||||
manifest := cbsManifest{Updates: make([]string, 0, blockCount)}
|
||||
for i := 0; i < blockCount; i++ {
|
||||
_, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(blockKey(i)),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader(blockBody(i)),
|
||||
})
|
||||
require.NoError(t, err, "upload block %d", i)
|
||||
manifest.Updates = append(manifest.Updates, blockKey(i))
|
||||
}
|
||||
manifestBody, err := json.Marshal(manifest)
|
||||
require.NoError(t, err)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(manifestKey),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader(manifestBody),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Re-uploading the same database under the same manifest name must not
|
||||
// commit over generation 1.
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(manifestKey),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte("different manifest")),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err))
|
||||
|
||||
// Download: list the container, fetch the manifest, fetch and verify every
|
||||
// block it names.
|
||||
listOut, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, listOut.Contents, blockCount+1, "manifest plus %d blocks", blockCount)
|
||||
|
||||
manifestETag, rawManifest := getCBSTestObject(t, client, bucket, manifestKey)
|
||||
var fetched cbsManifest
|
||||
require.NoError(t, json.Unmarshal(rawManifest, &fetched))
|
||||
require.Len(t, fetched.Updates, blockCount)
|
||||
for i, key := range fetched.Updates {
|
||||
_, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, sha256.Sum256(blockBody(i)), sha256.Sum256(body), "block %s checksum", key)
|
||||
}
|
||||
|
||||
// Checkpoint: append a new block, rewrite the manifest by CAS on the
|
||||
// fetched ETag. A reader racing us with the old identifier loses.
|
||||
newBlock := fmt.Sprintf("%sblock-%06d", prefix, blockCount)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(newBlock),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader(blockBody(blockCount)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
updated := fetched
|
||||
updated.Updates = append(updated.Updates, newBlock)
|
||||
updatedBody, err := json.Marshal(updated)
|
||||
require.NoError(t, err)
|
||||
|
||||
// A writer holding the generation-1 identifier loses the race...
|
||||
stale, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(manifestKey),
|
||||
IfMatch: aws.String("\"stale-generation-etag\""),
|
||||
Body: bytes.NewReader(updatedBody),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, isPreconditionFailed(err))
|
||||
assert.Nil(t, stale)
|
||||
|
||||
// ...while the writer that re-read the manifest commits.
|
||||
gen2, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(manifestKey),
|
||||
IfMatch: aws.String(manifestETag),
|
||||
Body: bytes.NewReader(updatedBody),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, gen2.ETag)
|
||||
|
||||
// The committed database is consistent: manifest lists every block and
|
||||
// every block fetches intact.
|
||||
_, rawManifest = getCBSTestObject(t, client, bucket, manifestKey)
|
||||
require.NoError(t, json.Unmarshal(rawManifest, &fetched))
|
||||
require.Len(t, fetched.Updates, blockCount+1)
|
||||
for i, key := range fetched.Updates[:blockCount] {
|
||||
_, body := getCBSTestObject(t, client, bucket, key)
|
||||
assert.Equal(t, sha256.Sum256(blockBody(i)), sha256.Sum256(body))
|
||||
}
|
||||
_, body := getCBSTestObject(t, client, bucket, newBlock)
|
||||
assert.Equal(t, sha256.Sum256(blockBody(blockCount)), sha256.Sum256(body))
|
||||
|
||||
// Tear the database down the way bcv destroy does.
|
||||
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(manifestKey)})
|
||||
require.NoError(t, err)
|
||||
deleteOut, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Delete: &types.Delete{Objects: []types.ObjectIdentifier{
|
||||
{Key: aws.String(blockKey(0))},
|
||||
{Key: aws.String(newBlock)},
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, deleteOut.Deleted, 2)
|
||||
}
|
||||
|
||||
func startMultipartUpload(t *testing.T, client *s3.Client, bucket, key, body string) (uploadID string, partETag string) {
|
||||
t.Helper()
|
||||
ctx, cancel := cbsTestContext()
|
||||
defer cancel()
|
||||
|
||||
createOut, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, createOut.UploadId)
|
||||
|
||||
partOut, err := client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: createOut.UploadId,
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader([]byte(body)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, partOut.ETag)
|
||||
|
||||
return aws.ToString(createOut.UploadId), aws.ToString(partOut.ETag)
|
||||
}
|
||||
|
||||
func abortMultipartUpload(client *s3.Client, bucket, key, uploadID string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, _ = client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user