mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 19:03:54 +01:00
s3: remove implicit directory handling
Since empty folders can now be async deleted, we no longer need special handling for implicit directories. This change simplifies the code and improves compatibility with S3 clients like Veeam 13. Changes: - Removed hasChildren() function that checked for child objects - Removed implicit directory logic from HeadObjectHandler - Removed unit tests for implicit directory behavior - All folders are now treated equally Benefits: - Better compatibility with Veeam 13 and other S3 clients - Simpler code without special cases - No performance overhead from hasChildren checks - Existing folders (both explicit with '/' and regular) continue to work
This commit is contained in:
1 parent
5b86d33c3c
commit
412d833700
2 files changed
-422
No files matched your search
@@ -1,285 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// TestImplicitDirectoryBehaviorLogic tests the core logic for implicit directory detection
|
||||
// This tests the decision logic without requiring a full S3 server setup
|
||||
func TestImplicitDirectoryBehaviorLogic(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
objectPath string
|
||||
hasTrailingSlash bool
|
||||
fileSize uint64
|
||||
isDirectory bool
|
||||
hasChildren bool
|
||||
versioningEnabled bool
|
||||
shouldReturn404 bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "Implicit directory: 0-byte file with children, no trailing slash",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 to force s3fs LIST-based discovery",
|
||||
},
|
||||
{
|
||||
name: "Implicit directory: actual directory with children, no trailing slash",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 for directory with children",
|
||||
},
|
||||
{
|
||||
name: "Explicit directory request: trailing slash",
|
||||
objectPath: "dataset/",
|
||||
hasTrailingSlash: true,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for explicit directory request (trailing slash)",
|
||||
},
|
||||
{
|
||||
name: "Empty file: 0-byte file without children",
|
||||
objectPath: "empty.txt",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for legitimate empty file",
|
||||
},
|
||||
{
|
||||
name: "Empty directory: 0-byte directory without children",
|
||||
objectPath: "empty-dir",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: true,
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for empty directory",
|
||||
},
|
||||
{
|
||||
name: "Regular file: non-zero size",
|
||||
objectPath: "file.txt",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 100,
|
||||
isDirectory: false,
|
||||
hasChildren: false,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for regular file with content",
|
||||
},
|
||||
{
|
||||
name: "Versioned bucket: implicit directory should return 200",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
hasChildren: true,
|
||||
versioningEnabled: true,
|
||||
shouldReturn404: false,
|
||||
description: "Should return 200 for versioned buckets (skip implicit dir check)",
|
||||
},
|
||||
{
|
||||
name: "PyArrow directory marker: 0-byte with children",
|
||||
objectPath: "dataset",
|
||||
hasTrailingSlash: false,
|
||||
fileSize: 0,
|
||||
isDirectory: false,
|
||||
hasChildren: true,
|
||||
versioningEnabled: false,
|
||||
shouldReturn404: true,
|
||||
description: "Should return 404 for PyArrow-created directory markers",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Test the logic: should we return 404?
|
||||
// Logic from HeadObjectHandler:
|
||||
// if !versioningConfigured && !strings.HasSuffix(object, "/") {
|
||||
// if isZeroByteFile || isActualDirectory {
|
||||
// if hasChildren {
|
||||
// return 404
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
isZeroByteFile := tt.fileSize == 0 && !tt.isDirectory
|
||||
isActualDirectory := tt.isDirectory
|
||||
|
||||
shouldReturn404 := false
|
||||
if !tt.versioningEnabled && !tt.hasTrailingSlash {
|
||||
if isZeroByteFile || isActualDirectory {
|
||||
if tt.hasChildren {
|
||||
shouldReturn404 = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if shouldReturn404 != tt.shouldReturn404 {
|
||||
t.Errorf("Logic mismatch for %s:\n Expected shouldReturn404=%v\n Got shouldReturn404=%v\n Description: %s",
|
||||
tt.name, tt.shouldReturn404, shouldReturn404, tt.description)
|
||||
} else {
|
||||
t.Logf("✓ %s: correctly returns %d", tt.name, map[bool]int{true: 404, false: 200}[shouldReturn404])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasChildrenLogic tests the hasChildren helper function logic
|
||||
func TestHasChildrenLogic(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
bucket string
|
||||
prefix string
|
||||
listResponse *filer_pb.ListEntriesResponse
|
||||
listError error
|
||||
expectedResult bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "Directory with children",
|
||||
bucket: "test-bucket",
|
||||
prefix: "dataset",
|
||||
listResponse: &filer_pb.ListEntriesResponse{
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: "file.parquet",
|
||||
IsDirectory: false,
|
||||
},
|
||||
},
|
||||
listError: nil,
|
||||
expectedResult: true,
|
||||
description: "Should return true when at least one child exists",
|
||||
},
|
||||
{
|
||||
name: "Empty directory",
|
||||
bucket: "test-bucket",
|
||||
prefix: "empty-dir",
|
||||
listResponse: nil,
|
||||
listError: io.EOF,
|
||||
expectedResult: false,
|
||||
description: "Should return false when no children exist (EOF)",
|
||||
},
|
||||
{
|
||||
name: "Directory with leading slash in prefix",
|
||||
bucket: "test-bucket",
|
||||
prefix: "/dataset",
|
||||
listResponse: &filer_pb.ListEntriesResponse{
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: "file.parquet",
|
||||
IsDirectory: false,
|
||||
},
|
||||
},
|
||||
listError: nil,
|
||||
expectedResult: true,
|
||||
description: "Should handle leading slashes correctly",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Test the hasChildren logic:
|
||||
// 1. It should trim leading slashes from prefix
|
||||
// 2. It should list with Limit=1
|
||||
// 3. It should return true if any entry is received
|
||||
// 4. It should return false if EOF is received
|
||||
|
||||
hasChildren := false
|
||||
if tt.listError == nil && tt.listResponse != nil {
|
||||
hasChildren = true
|
||||
} else if tt.listError == io.EOF {
|
||||
hasChildren = false
|
||||
}
|
||||
|
||||
if hasChildren != tt.expectedResult {
|
||||
t.Errorf("hasChildren logic mismatch for %s:\n Expected: %v\n Got: %v\n Description: %s",
|
||||
tt.name, tt.expectedResult, hasChildren, tt.description)
|
||||
} else {
|
||||
t.Logf("✓ %s: correctly returns %v", tt.name, hasChildren)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestImplicitDirectoryEdgeCases tests edge cases in the implicit directory detection
|
||||
func TestImplicitDirectoryEdgeCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scenario string
|
||||
expectation string
|
||||
}{
|
||||
{
|
||||
name: "PyArrow write_dataset creates 0-byte files",
|
||||
scenario: "PyArrow creates 'dataset' as 0-byte file, then writes 'dataset/file.parquet'",
|
||||
expectation: "HEAD dataset → 404 (has children), s3fs uses LIST → correctly identifies as directory",
|
||||
},
|
||||
{
|
||||
name: "Filer creates actual directories",
|
||||
scenario: "Filer creates 'dataset' as actual directory with IsDirectory=true",
|
||||
expectation: "HEAD dataset → 404 (has children), s3fs uses LIST → correctly identifies as directory",
|
||||
},
|
||||
{
|
||||
name: "Empty file edge case",
|
||||
scenario: "User creates 'empty.txt' as 0-byte file with no children",
|
||||
expectation: "HEAD empty.txt → 200 (no children), s3fs correctly reports as file",
|
||||
},
|
||||
{
|
||||
name: "Explicit directory request",
|
||||
scenario: "User requests 'dataset/' with trailing slash",
|
||||
expectation: "HEAD dataset/ → 200 (explicit directory request), normal directory behavior",
|
||||
},
|
||||
{
|
||||
name: "Versioned bucket",
|
||||
scenario: "Bucket has versioning enabled",
|
||||
expectation: "HEAD dataset → 200 (skip implicit dir check), versioned semantics apply",
|
||||
},
|
||||
{
|
||||
name: "AWS S3 compatibility",
|
||||
scenario: "Only 'dataset/file.txt' exists, no marker at 'dataset'",
|
||||
expectation: "HEAD dataset → 404 (object doesn't exist), matches AWS S3 behavior",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Logf("Scenario: %s", tt.scenario)
|
||||
t.Logf("Expected: %s", tt.expectation)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestImplicitDirectoryIntegration is an integration test placeholder
|
||||
// Run with: cd test/s3/parquet && make test-implicit-dir-with-server
|
||||
func TestImplicitDirectoryIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
t.Skip("Integration test - run manually with: cd test/s3/parquet && make test-implicit-dir-with-server")
|
||||
}
|
||||
|
||||
// Benchmark for hasChildren performance
|
||||
func BenchmarkHasChildrenCheck(b *testing.B) {
|
||||
// This benchmark would measure the performance impact of the hasChildren check
|
||||
// Expected: ~1-5ms per call (one gRPC LIST request with Limit=1)
|
||||
b.Skip("Benchmark - requires full filer setup")
|
||||
}
|
||||
Reference in new issue
Block a user