diff --git a/test/s3/copying/s3_copying_test.go b/test/s3/copying/s3_copying_test.go index 6b46f6802..bfa8ba7d4 100644 --- a/test/s3/copying/s3_copying_test.go +++ b/test/s3/copying/s3_copying_test.go @@ -7,8 +7,10 @@ import ( "fmt" "io" mathrand "math/rand" + "net/http" "net/url" "os" + "strconv" "strings" "testing" "time" @@ -24,24 +26,26 @@ import ( // S3TestConfig holds configuration for S3 tests type S3TestConfig struct { - Endpoint string - AccessKey string - SecretKey string - Region string - BucketPrefix string - UseSSL bool - SkipVerifySSL bool + Endpoint string + MasterEndpoint string + AccessKey string + SecretKey string + Region string + BucketPrefix string + UseSSL bool + SkipVerifySSL bool } // Default test configuration - should match test_config.json var defaultConfig = &S3TestConfig{ - Endpoint: "http://127.0.0.1:8000", // Use explicit IPv4 address - AccessKey: "some_access_key1", - SecretKey: "some_secret_key1", - Region: "us-east-1", - BucketPrefix: "test-copying-", - UseSSL: false, - SkipVerifySSL: true, + Endpoint: "http://127.0.0.1:8000", // Use explicit IPv4 address + MasterEndpoint: "http://127.0.0.1:9333", // Default SeaweedFS master HTTP port + AccessKey: "some_access_key1", + SecretKey: "some_secret_key1", + Region: "us-east-1", + BucketPrefix: "test-copying-", + UseSSL: false, + SkipVerifySSL: true, } func init() { @@ -49,6 +53,9 @@ func init() { if endpoint := os.Getenv("S3_ENDPOINT"); endpoint != "" { defaultConfig.Endpoint = endpoint } + if masterEndpoint := os.Getenv("MASTER_ENDPOINT"); masterEndpoint != "" { + defaultConfig.MasterEndpoint = masterEndpoint + } } // getS3Client creates an AWS S3 client for testing @@ -95,10 +102,20 @@ func getNewBucketName() string { timestamp := time.Now().UnixNano() // Add random suffix to prevent collisions when tests run quickly randomSuffix := mathrand.Intn(100000) - return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix) + return fmt.Sprintf("%sr%s-%d-%d", defaultConfig.BucketPrefix, testRunID, timestamp, randomSuffix) } -// cleanupTestBuckets removes any leftover test buckets from previous runs +// testRunID uniquely identifies this `go test` invocation. It's embedded into +// every bucket created by getNewBucketName (after defaultConfig.BucketPrefix), so +// the cleanup sweep can scope deletions to buckets that belong to this run only — +// letting concurrent runs against the same endpoints coexist without tearing each +// other's buckets down. Stale buckets left behind by a crashed prior run share the +// family prefix but a different runID, so they are not swept here; `make clean` +// (or wiping the data dir) is the right cleanup for that case. +var testRunID = strconv.FormatInt(time.Now().UnixNano(), 36) + +// cleanupTestBuckets removes any leftover test buckets from this run. +// Buckets from concurrent runs carry a different runID marker and are skipped. func cleanupTestBuckets(t *testing.T, client *s3.Client) { resp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) if err != nil { @@ -106,10 +123,11 @@ func cleanupTestBuckets(t *testing.T, client *s3.Client) { return } + runPrefix := defaultConfig.BucketPrefix + "r" + testRunID + "-" for _, bucket := range resp.Buckets { bucketName := *bucket.Name - // Only delete buckets that match our test prefix - if strings.HasPrefix(bucketName, defaultConfig.BucketPrefix) { + // Only sweep buckets owned by this run (prefix + runID marker). + if strings.HasPrefix(bucketName, runPrefix) { t.Logf("Cleaning up leftover test bucket: %s", bucketName) deleteBucket(t, client, bucketName) } @@ -118,6 +136,11 @@ func cleanupTestBuckets(t *testing.T, client *s3.Client) { // createBucket creates a new bucket for testing func createBucket(t *testing.T, client *s3.Client, bucketName string) { + // Sweep stale buckets from prior tests/runs so each new bucket starts on a + // fresh slate. Without this, leaked collection volumes from a panicked or + // interrupted earlier run accumulate on a single `weed mini` data node and + // the suite eventually exhausts its volume slots. + cleanupTestBuckets(t, client) // First, try to delete the bucket if it exists (cleanup from previous failed tests) deleteBucket(t, client, bucketName) @@ -128,8 +151,16 @@ func createBucket(t *testing.T, client *s3.Client, bucketName string) { require.NoError(t, err) } -// deleteBucket deletes a bucket and all its contents +// deleteBucket deletes a bucket and all its contents. +// Always force-drops the underlying collection at the master afterwards: the S3 +// DeleteBucket can race with concurrent `volume_grow` requests (the warm-create +// batch keeps registering volumes after the master's collection-delete sweep has +// already snapshotted the layout), so 1-3 volumes per bucket can leak. Without +// this, running enough tests on a single `weed mini` server exhausts the data +// node's volume slots and every subsequent PutObject 500s with "Not enough data +// nodes found". func deleteBucket(t *testing.T, client *s3.Client, bucketName string) { + defer forceDeleteCollection(t, bucketName) // First, delete all objects deleteAllObjects(t, client, bucketName) @@ -145,6 +176,39 @@ func deleteBucket(t *testing.T, client *s3.Client, bucketName string) { } } +// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's +// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but +// in-flight `volume_grow` requests can register volumes after the master's first sweep, leaking +// them. Best-effort: a 400 from the master means the collection was already gone, which is the +// success path and not an error. +func forceDeleteCollection(t *testing.T, bucketName string) { + if defaultConfig.MasterEndpoint == "" { + return + } + endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err) + return + } + httpClient := &http.Client{Timeout: 5 * time.Second} + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Note: force-delete collection %s failed: %v", bucketName, err) + return + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + switch resp.StatusCode { + case http.StatusNoContent: + t.Logf("Force-deleted collection %s", bucketName) + case http.StatusBadRequest: + // Collection already gone - normal path when DeleteBucket succeeded. + default: + t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode) + } +} + // deleteAllObjects deletes all objects in a bucket func deleteAllObjects(t *testing.T, client *s3.Client, bucketName string) { // List all objects diff --git a/test/s3/cors/s3_cors_test.go b/test/s3/cors/s3_cors_test.go index 0155254e1..e4c35d442 100644 --- a/test/s3/cors/s3_cors_test.go +++ b/test/s3/cors/s3_cors_test.go @@ -3,6 +3,11 @@ package cors import ( "context" "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" "strings" "testing" "time" @@ -19,26 +24,52 @@ import ( // S3TestConfig holds configuration for S3 tests type S3TestConfig struct { - Endpoint string - AccessKey string - SecretKey string - Region string - BucketPrefix string - UseSSL bool - SkipVerifySSL bool + Endpoint string + MasterEndpoint string + AccessKey string + SecretKey string + Region string + BucketPrefix string + UseSSL bool + SkipVerifySSL bool } +// allTestBucketPrefixes lists every prefix used to name buckets in this test suite. +// cleanupAllTestBuckets uses it to find stale buckets from prior tests/runs. +// Add the new prefix here whenever a test introduces one. +var allTestBucketPrefixes = []string{ + "test-cors-", +} + +// testRunID uniquely identifies this `go test` invocation. It's embedded into +// every bucket created by createTestBucket (after defaultConfig.BucketPrefix), so +// the cleanup sweep can scope deletions to buckets that belong to this run only — +// letting concurrent runs against the same endpoints coexist without tearing each +// other's buckets down. Stale buckets left behind by a crashed prior run share the +// family prefix but a different runID, so they are not swept here; `make clean` +// (or wiping the data dir) is the right cleanup for that case. +var testRunID = strconv.FormatInt(time.Now().UnixNano(), 36) + // getDefaultConfig returns a fresh instance of the default test configuration // to avoid parallel test issues with global mutable state func getDefaultConfig() *S3TestConfig { + endpoint := os.Getenv("S3_ENDPOINT") + if endpoint == "" { + endpoint = "http://localhost:8333" // Default SeaweedFS S3 port + } + masterEndpoint := os.Getenv("MASTER_ENDPOINT") + if masterEndpoint == "" { + masterEndpoint = "http://127.0.0.1:9333" // Default SeaweedFS master HTTP port + } return &S3TestConfig{ - Endpoint: "http://localhost:8333", // Default SeaweedFS S3 port - AccessKey: "some_access_key1", - SecretKey: "some_secret_key1", - Region: "us-east-1", - BucketPrefix: "test-cors-", - UseSSL: false, - SkipVerifySSL: true, + Endpoint: endpoint, + MasterEndpoint: masterEndpoint, + AccessKey: "some_access_key1", + SecretKey: "some_secret_key1", + Region: "us-east-1", + BucketPrefix: "test-cors-", + UseSSL: false, + SkipVerifySSL: true, } } @@ -71,7 +102,12 @@ func getS3Client(t *testing.T) *s3.Client { // createTestBucket creates a test bucket with a unique name func createTestBucket(t *testing.T, client *s3.Client) string { defaultConfig := getDefaultConfig() - bucketName := fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, time.Now().UnixNano()) + // Sweep stale buckets from prior tests in this run so each new bucket starts + // on a fresh slate. Without this, leaked collection volumes accumulate on a + // single `weed mini` data node and the suite eventually exhausts its volume + // slots. + cleanupAllTestBuckets(t, client) + bucketName := fmt.Sprintf("%sr%s-%d", defaultConfig.BucketPrefix, testRunID, time.Now().UnixNano()) _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ Bucket: aws.String(bucketName), @@ -84,8 +120,16 @@ func createTestBucket(t *testing.T, client *s3.Client) string { return bucketName } -// cleanupTestBucket removes the test bucket and all its contents +// cleanupTestBucket removes the test bucket and all its contents. +// Always force-drops the underlying collection at the master afterwards: the S3 +// DeleteBucket can race with concurrent `volume_grow` requests (the warm-create +// batch keeps registering volumes after the master's collection-delete sweep has +// already snapshotted the layout), so 1-3 volumes per bucket can leak. Without +// this, running enough tests on a single `weed mini` server exhausts the data +// node's volume slots and every subsequent PutObject 500s with "Not enough data +// nodes found". func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { + defer forceDeleteCollection(t, bucketName) // First, delete all objects in the bucket listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ Bucket: aws.String(bucketName), @@ -111,6 +155,67 @@ func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { } } +// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's +// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but +// in-flight `volume_grow` requests can register volumes after the master's first sweep, leaking +// them. Best-effort: a 400 from the master means the collection was already gone, which is the +// success path and not an error. +func forceDeleteCollection(t *testing.T, bucketName string) { + masterEndpoint := getDefaultConfig().MasterEndpoint + if masterEndpoint == "" { + return + } + endpoint := strings.TrimRight(masterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err) + return + } + httpClient := &http.Client{Timeout: 5 * time.Second} + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Note: force-delete collection %s failed: %v", bucketName, err) + return + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + switch resp.StatusCode { + case http.StatusNoContent: + t.Logf("Force-deleted collection %s", bucketName) + case http.StatusBadRequest: + // Collection already gone - normal path when DeleteBucket succeeded. + default: + t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode) + } +} + +// cleanupAllTestBuckets cleans up any leftover test buckets owned by this run +// (prefix + runID marker). Called from createTestBucket before each new bucket +// creation so a single `weed mini` data node does not exhaust its volume slots +// after many tests. +func cleanupAllTestBuckets(t *testing.T, client *s3.Client) { + listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) + if err != nil { + t.Logf("Warning: failed to list buckets for cleanup: %v", err) + return + } + + for _, bucket := range listResp.Buckets { + if bucket.Name == nil { + continue + } + for _, prefix := range allTestBucketPrefixes { + // Only sweep buckets owned by this run (prefix + runID marker). + // Buckets from concurrent runs carry a different runID and are skipped. + if strings.HasPrefix(*bucket.Name, prefix+"r"+testRunID+"-") { + t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name) + cleanupTestBucket(t, client, *bucket.Name) + break + } + } + } +} + // TestCORSConfigurationManagement tests basic CORS configuration CRUD operations func TestCORSConfigurationManagement(t *testing.T) { client := getS3Client(t) diff --git a/test/s3/tagging/s3_tagging_test.go b/test/s3/tagging/s3_tagging_test.go index 4606ec800..28db4dded 100644 --- a/test/s3/tagging/s3_tagging_test.go +++ b/test/s3/tagging/s3_tagging_test.go @@ -3,7 +3,11 @@ package tagging import ( "context" "fmt" + "io" + "net/http" + "net/url" "os" + "strconv" "strings" "testing" "time" @@ -19,21 +23,42 @@ import ( // S3TestConfig holds configuration for S3 tests type S3TestConfig struct { - Endpoint string - AccessKey string - SecretKey string - Region string - BucketPrefix string - UseSSL bool - SkipVerifySSL bool + Endpoint string + MasterEndpoint string + AccessKey string + SecretKey string + Region string + BucketPrefix string + UseSSL bool + SkipVerifySSL bool } +// allTestBucketPrefixes lists every prefix used to name buckets in this test suite. +// cleanupAllTestBuckets uses it to find stale buckets from prior tests/runs. +// Add the new prefix here whenever a test introduces one. +var allTestBucketPrefixes = []string{ + "test-tagging-", +} + +// testRunID uniquely identifies this `go test` invocation. It's embedded into +// every bucket created by createTestBucket (after defaultConfig.BucketPrefix), so +// the cleanup sweep can scope deletions to buckets that belong to this run only — +// letting concurrent runs against the same endpoints coexist without tearing each +// other's buckets down. Stale buckets left behind by a crashed prior run share the +// family prefix but a different runID, so they are not swept here; `make clean` +// (or wiping the data dir) is the right cleanup for that case. +var testRunID = strconv.FormatInt(time.Now().UnixNano(), 36) + // getDefaultConfig returns a fresh instance of the default test configuration func getDefaultConfig() *S3TestConfig { endpoint := os.Getenv("S3_ENDPOINT") if endpoint == "" { endpoint = "http://localhost:8333" // Default SeaweedFS S3 port } + masterEndpoint := os.Getenv("MASTER_ENDPOINT") + if masterEndpoint == "" { + masterEndpoint = "http://127.0.0.1:9333" // Default SeaweedFS master HTTP port + } accessKey := os.Getenv("S3_ACCESS_KEY") if accessKey == "" { accessKey = "some_access_key1" @@ -43,13 +68,14 @@ func getDefaultConfig() *S3TestConfig { secretKey = "some_secret_key1" } return &S3TestConfig{ - Endpoint: endpoint, - AccessKey: accessKey, - SecretKey: secretKey, - Region: "us-east-1", - BucketPrefix: "test-tagging-", - UseSSL: false, - SkipVerifySSL: true, + Endpoint: endpoint, + MasterEndpoint: masterEndpoint, + AccessKey: accessKey, + SecretKey: secretKey, + Region: "us-east-1", + BucketPrefix: "test-tagging-", + UseSSL: false, + SkipVerifySSL: true, } } @@ -76,7 +102,12 @@ func getS3Client(t *testing.T) *s3.Client { // createTestBucket creates a test bucket with a unique name func createTestBucket(t *testing.T, client *s3.Client) string { defaultConfig := getDefaultConfig() - bucketName := fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, time.Now().UnixNano()) + // Sweep stale buckets from prior tests in this run so each new bucket starts + // on a fresh slate. Without this, leaked collection volumes accumulate on a + // single `weed mini` data node and the suite eventually exhausts its volume + // slots. + cleanupAllTestBuckets(t, client) + bucketName := fmt.Sprintf("%sr%s-%d", defaultConfig.BucketPrefix, testRunID, time.Now().UnixNano()) _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ Bucket: aws.String(bucketName), @@ -89,8 +120,16 @@ func createTestBucket(t *testing.T, client *s3.Client) string { return bucketName } -// cleanupTestBucket removes the test bucket and all its contents +// cleanupTestBucket removes the test bucket and all its contents. +// Always force-drops the underlying collection at the master afterwards: the S3 +// DeleteBucket can race with concurrent `volume_grow` requests (the warm-create +// batch keeps registering volumes after the master's collection-delete sweep has +// already snapshotted the layout), so 1-3 volumes per bucket can leak. Without +// this, running enough tests on a single `weed mini` server exhausts the data +// node's volume slots and every subsequent PutObject 500s with "Not enough data +// nodes found". func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { + defer forceDeleteCollection(t, bucketName) // First, delete all objects in the bucket listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ Bucket: aws.String(bucketName), @@ -143,6 +182,67 @@ func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { } } +// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's +// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but +// in-flight `volume_grow` requests can register volumes after the master's first sweep, leaking +// them. Best-effort: a 400 from the master means the collection was already gone, which is the +// success path and not an error. +func forceDeleteCollection(t *testing.T, bucketName string) { + masterEndpoint := getDefaultConfig().MasterEndpoint + if masterEndpoint == "" { + return + } + endpoint := strings.TrimRight(masterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err) + return + } + httpClient := &http.Client{Timeout: 5 * time.Second} + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Note: force-delete collection %s failed: %v", bucketName, err) + return + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + switch resp.StatusCode { + case http.StatusNoContent: + t.Logf("Force-deleted collection %s", bucketName) + case http.StatusBadRequest: + // Collection already gone - normal path when DeleteBucket succeeded. + default: + t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode) + } +} + +// cleanupAllTestBuckets cleans up any leftover test buckets owned by this run +// (prefix + runID marker). Called from createTestBucket before each new bucket +// creation so a single `weed mini` data node does not exhaust its volume slots +// after many tests. +func cleanupAllTestBuckets(t *testing.T, client *s3.Client) { + listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) + if err != nil { + t.Logf("Warning: failed to list buckets for cleanup: %v", err) + return + } + + for _, bucket := range listResp.Buckets { + if bucket.Name == nil { + continue + } + for _, prefix := range allTestBucketPrefixes { + // Only sweep buckets owned by this run (prefix + runID marker). + // Buckets from concurrent runs carry a different runID and are skipped. + if strings.HasPrefix(*bucket.Name, prefix+"r"+testRunID+"-") { + t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name) + cleanupTestBucket(t, client, *bucket.Name) + break + } + } + } +} + // TestObjectTaggingOnUpload tests that tags sent during object upload (via X-Amz-Tagging header) // are properly stored and can be retrieved. This is the fix for GitHub issue #7589. func TestObjectTaggingOnUpload(t *testing.T) { diff --git a/test/s3/versioning/s3_versioning_test.go b/test/s3/versioning/s3_versioning_test.go index cb8d72535..b4fecdb73 100644 --- a/test/s3/versioning/s3_versioning_test.go +++ b/test/s3/versioning/s3_versioning_test.go @@ -3,6 +3,11 @@ package s3api import ( "context" "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" "strings" "testing" "time" @@ -19,24 +24,59 @@ import ( // S3TestConfig holds configuration for S3 tests type S3TestConfig struct { - Endpoint string - AccessKey string - SecretKey string - Region string - BucketPrefix string - UseSSL bool - SkipVerifySSL bool + Endpoint string + MasterEndpoint string + AccessKey string + SecretKey string + Region string + BucketPrefix string + UseSSL bool + SkipVerifySSL bool } // Default test configuration - should match s3tests.conf var defaultConfig = &S3TestConfig{ - Endpoint: "http://localhost:8333", // Default SeaweedFS S3 port - AccessKey: "some_access_key1", - SecretKey: "some_secret_key1", - Region: "us-east-1", - BucketPrefix: "test-versioning-", - UseSSL: false, - SkipVerifySSL: true, + Endpoint: firstNonEmpty(os.Getenv("S3_ENDPOINT"), "http://localhost:8333"), // Default SeaweedFS S3 port + MasterEndpoint: firstNonEmpty(os.Getenv("MASTER_ENDPOINT"), "http://127.0.0.1:9333"), // Default SeaweedFS master HTTP port + AccessKey: "some_access_key1", + SecretKey: "some_secret_key1", + Region: "us-east-1", + BucketPrefix: "test-versioning-", + UseSSL: false, + SkipVerifySSL: true, +} + +// allTestBucketPrefixes lists every prefix used to name buckets in this test suite. +// cleanupAllTestBuckets uses it to find stale buckets from prior tests/runs. +// Add the new prefix here whenever a test introduces one. +var allTestBucketPrefixes = []string{ + "test-versioning-", // covers test-versioning-, test-versioning-directories, test-versioning-interaction-, test-versioned-acl/list + "test-versioned-", + "test-error-messages-", + "test-delete-markers", + "test-concurrent-delete", + "test-suspended-versioning-delete", + "test-bucket-", +} + +// testRunID uniquely identifies this `go test` invocation. It's embedded into +// every bucket created by getNewBucketName (after defaultConfig.BucketPrefix), so +// the cleanup sweep can scope deletions to buckets that belong to this run only — +// letting concurrent runs against the same endpoints coexist without tearing each +// other's buckets down. Stale buckets left behind by a crashed prior run share the +// family prefix but a different runID, so they are not swept here; `make clean` +// (or wiping the data dir) is the right cleanup for that case. Fixed-name buckets +// like "test-versioning-directories" do not flow through this sweep — they are +// short-lived and handled by their own deferred deleteBucket. +var testRunID = strconv.FormatInt(time.Now().UnixNano(), 36) + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" } // getS3Client creates an AWS S3 client for testing @@ -67,19 +107,32 @@ func getS3Client(t *testing.T) *s3.Client { // getNewBucketName generates a unique bucket name func getNewBucketName() string { timestamp := time.Now().UnixNano() - return fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, timestamp) + return fmt.Sprintf("%sr%s-%d", defaultConfig.BucketPrefix, testRunID, timestamp) } // createBucket creates a new bucket for testing func createBucket(t *testing.T, client *s3.Client, bucketName string) { + // Sweep stale buckets from prior tests in this run so each new bucket starts + // on a fresh slate. Without this, leaked collection volumes accumulate on a + // single `weed mini` data node and the suite eventually exhausts its volume + // slots. + cleanupAllTestBuckets(t, client) _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ Bucket: aws.String(bucketName), }) require.NoError(t, err) } -// deleteBucket deletes a bucket and all its contents +// deleteBucket deletes a bucket and all its contents. +// Always force-drops the underlying collection at the master afterwards: the S3 +// DeleteBucket can race with concurrent `volume_grow` requests (the warm-create +// batch keeps registering volumes after the master's collection-delete sweep has +// already snapshotted the layout), so 1-3 volumes per bucket can leak. Without +// this, running enough tests on a single `weed mini` server exhausts the data +// node's volume slots and every subsequent PutObject 500s with "Not enough data +// nodes found". func deleteBucket(t *testing.T, client *s3.Client, bucketName string) { + defer forceDeleteCollection(t, bucketName) // First, delete all objects and versions err := deleteAllObjectVersions(t, client, bucketName) if err != nil { @@ -95,6 +148,66 @@ func deleteBucket(t *testing.T, client *s3.Client, bucketName string) { } } +// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's +// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but +// in-flight `volume_grow` requests can register volumes after the master's first sweep, leaking +// them. Best-effort: a 400 from the master means the collection was already gone, which is the +// success path and not an error. +func forceDeleteCollection(t *testing.T, bucketName string) { + if defaultConfig.MasterEndpoint == "" { + return + } + endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err) + return + } + httpClient := &http.Client{Timeout: 5 * time.Second} + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("Note: force-delete collection %s failed: %v", bucketName, err) + return + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + switch resp.StatusCode { + case http.StatusNoContent: + t.Logf("Force-deleted collection %s", bucketName) + case http.StatusBadRequest: + // Collection already gone - normal path when DeleteBucket succeeded. + default: + t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode) + } +} + +// cleanupAllTestBuckets cleans up any leftover test buckets owned by this run +// (prefix + runID marker). Called from createBucket before each new bucket +// creation so a single `weed mini` data node does not exhaust its volume slots +// after many tests. +func cleanupAllTestBuckets(t *testing.T, client *s3.Client) { + listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{}) + if err != nil { + t.Logf("Warning: failed to list buckets for cleanup: %v", err) + return + } + + for _, bucket := range listResp.Buckets { + if bucket.Name == nil { + continue + } + for _, prefix := range allTestBucketPrefixes { + // Only sweep buckets owned by this run (prefix + runID marker). + // Buckets from concurrent runs carry a different runID and are skipped. + if strings.HasPrefix(*bucket.Name, prefix+"r"+testRunID+"-") { + t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name) + deleteBucket(t, client, *bucket.Name) + break + } + } + } +} + // deleteAllObjectVersions deletes all object versions in a bucket func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string) error { // List all object versions