mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-11 17:10:40 +02:00
* feat(s3/lifecycle/router): emit ABORT_MPU events for .uploads/<id> init dirs Detect a meta-log event at exactly .uploads/<upload_id> (a directory) and build the ObjectInfo from its destination key (entry.Extended[key]) so a rule with Filter.Prefix=foo/ matches an MPU uploading to foo/bar. Sub-events under .uploads/<id>/<part> ride a different mtime and would over-fire the ABORT_MPU schedule, so they're rejected explicitly. m.ObjectKey stays as ev.Key (.uploads/<upload_id>) — the dispatcher needs the upload directory path, not the destination key, to actually remove the in-flight upload. * feat(s3api): wire LifecycleDelete ABORT_MPU to remove the upload dir Replaces the retryLater stub. Validates the .uploads/<upload_id> shape of req.ObjectPath (so a malformed event can't escalate to a wider rm), then deletes the upload directory under <bucket>/.uploads/<id>. Maps NotFound to NOOP_RESOLVED, transport errors to RETRY_LATER, success to DONE. * refactor(s3api): drop redundant exists check before lifecycle ABORT_MPU rm s3a.rm already does a NotFound-returning lookup, so the pre-check just adds a round-trip. Map filer_pb.ErrNotFound to NOOP_RESOLVED on rm, keep transport errors as RETRY_LATER. * refactor(s3/lifecycle/router): use s3_constants for MPU paths + Extended key Drop the hardcoded ".uploads/" and "key" string literals; the symbols already exist as s3_constants.MultipartUploadsFolder and ExtMultipartObjectKey, and the server side reaches them through the same constants. Keeping the test helpers tied to those names also makes the negative-result tests meaningful — they'd otherwise still pass if the lookup constant drifted. * fix(s3api): close lifecycle ABORT_MPU traversal + NOT_FOUND gaps Two issues with the recent ABORT_MPU plumbing: - "." and ".." passed the no-slash check but resolve to the bucket root via util.JoinPath, so .uploads/.. could rm the wrong directory. - filer.DeleteEntry suppresses ErrNotFound and returns success, so the rm path can't distinguish missing from deleted; the previous version reported DONE for an already-aborted upload instead of NOOP_RESOLVED. Reject the two reserved names explicitly and restore the existence pre-check so the outcome map stays correct. Add a table-test covering the rejected paths. * fix(s3/lifecycle/bootstrap): walk MPU init dirs by destination key A real MPU init record is a directory under .uploads/<id> created by mkdir; the bootstrap walker was skipping every directory entry, so an MPU that existed before the meta-log subscription was never aborted. Even with the skip relaxed, MatchPath used the .uploads/<id> path, so a rule with Filter.Prefix=logs/ would never fire on an MPU uploading to logs/foo.txt. Add Entry.DestKey, let IsMPUInit directories through, and use DestKey for both MatchPath and ObjectInfo.Key. A bare init directory with no DestKey means metadata hasn't landed yet — skip rather than guess. * fix(s3/lifecycle): gate (kind, info) shape so MPU init only fires ABORT_MPU An MPU init record carries IsMPUInit=true and IsLatest=false. Without gating, the router and bootstrap walker matched it against every active ActionKey for the bucket, so NONCURRENT_DAYS / NEWER_NONCURRENT fired (IsLatest=false reads as a noncurrent version). The dispatcher would then BLOCK on empty version_id and freeze the cursor. Add a shape gate at both call sites: - IsMPUInit + non-ABORT_MPU kind → continue - regular object + ABORT_MPU kind → continue Plus a defense-in-depth check at the top of EvaluateAction so future callers can't reintroduce the bug. Tests cover all three layers. * test(s3/lifecycle): tighten dual-action coverage at the call sites - Walk multi-action: replace the kinds-as-set check with an exact-shape DeepEqual on (path, kind) tuples. The set check would have missed an MPU init wrongly firing NONCURRENT_DAYS — exactly the regression the (kind, info) gate fixes. - Router: add a converse case for the dual ExpirationDays + AbortIncompleteMultipartUpload rule. A regular current-version object must fire only EXPIRATION_DAYS; without the gate the dispatcher would also receive ABORT_MPU and rm the object via the MPU code path.
150 lines
5.1 KiB
Go
150 lines
5.1 KiB
Go
package s3api
|
|
|
|
import (
|
|
"bytes"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
)
|
|
|
|
func TestComputeEntryIdentity_BasicFields(t *testing.T) {
|
|
entry := &filer_pb.Entry{
|
|
Attributes: &filer_pb.FuseAttributes{Mtime: 1700000000, MtimeNs: 123, FileSize: 4096},
|
|
Chunks: []*filer_pb.FileChunk{
|
|
{FileId: "1,abc"},
|
|
{FileId: "1,def"},
|
|
},
|
|
}
|
|
id := computeEntryIdentity(entry)
|
|
want := int64(1700000000)*int64(1e9) + int64(123)
|
|
if id.MtimeNs != want {
|
|
t.Fatalf("MtimeNs want %d, got %d", want, id.MtimeNs)
|
|
}
|
|
if id.Size != 4096 {
|
|
t.Fatalf("Size want 4096, got %d", id.Size)
|
|
}
|
|
if id.HeadFid != "1,abc" {
|
|
t.Fatalf("HeadFid want 1,abc, got %s", id.HeadFid)
|
|
}
|
|
}
|
|
|
|
func TestComputeEntryIdentity_NilSafeMissingChunks(t *testing.T) {
|
|
if got := computeEntryIdentity(nil); got != nil {
|
|
t.Fatalf("nil entry should return nil, got %v", got)
|
|
}
|
|
id := computeEntryIdentity(&filer_pb.Entry{})
|
|
if id == nil {
|
|
t.Fatalf("entry with nil Attributes should still produce identity")
|
|
}
|
|
if id.HeadFid != "" {
|
|
t.Fatalf("missing chunks should yield empty HeadFid, got %s", id.HeadFid)
|
|
}
|
|
}
|
|
|
|
func TestHashExtended_OrderStable(t *testing.T) {
|
|
a := map[string][]byte{"k1": []byte("v1"), "k2": []byte("v2")}
|
|
b := map[string][]byte{"k2": []byte("v2"), "k1": []byte("v1")}
|
|
if !bytes.Equal(s3lifecycle.HashExtended(a), s3lifecycle.HashExtended(b)) {
|
|
t.Fatalf("hash should be insensitive to map iteration order")
|
|
}
|
|
}
|
|
|
|
func TestHashExtended_DelimiterCollisionResistant(t *testing.T) {
|
|
// Naively concatenated: "k1=v1k2v2" could collide with "k1=v1k" / "2v2".
|
|
// Length-prefix encoding must keep them apart.
|
|
a := map[string][]byte{"k1": []byte("v1"), "k2": []byte("v2")}
|
|
b := map[string][]byte{"k1": []byte("v1k2v2")}
|
|
if bytes.Equal(s3lifecycle.HashExtended(a), s3lifecycle.HashExtended(b)) {
|
|
t.Fatalf("delimiter-forged Extended payloads must not collide")
|
|
}
|
|
}
|
|
|
|
func TestHashExtended_NilEqualsEmpty(t *testing.T) {
|
|
if got := s3lifecycle.HashExtended(nil); len(got) != 0 {
|
|
t.Fatalf("nil should produce zero-length hash, got %d bytes", len(got))
|
|
}
|
|
if got := s3lifecycle.HashExtended(map[string][]byte{}); len(got) != 0 {
|
|
t.Fatalf("empty map should produce zero-length hash, got %d bytes", len(got))
|
|
}
|
|
}
|
|
|
|
func TestIdentityMatches_NilWantTreatedAsMatch(t *testing.T) {
|
|
// Bootstrap callers that don't yet have an identity to CAS against
|
|
// pass nil expected_identity; the server treats this as "no CAS".
|
|
live := &s3_lifecycle_pb.EntryIdentity{MtimeNs: 1, Size: 2}
|
|
if !identityMatches(live, nil) {
|
|
t.Fatalf("nil want should match")
|
|
}
|
|
}
|
|
|
|
func TestIdentityMatches_NilLiveDoesNotMatch(t *testing.T) {
|
|
if identityMatches(nil, &s3_lifecycle_pb.EntryIdentity{MtimeNs: 1}) {
|
|
t.Fatalf("nil live should not match a populated want")
|
|
}
|
|
}
|
|
|
|
func TestIdentityMatches_AllFieldsCompared(t *testing.T) {
|
|
base := &s3_lifecycle_pb.EntryIdentity{MtimeNs: 100, Size: 2048, HeadFid: "1,abc", ExtendedHash: []byte{0x01, 0x02}}
|
|
cases := []struct {
|
|
name string
|
|
live *s3_lifecycle_pb.EntryIdentity
|
|
want bool
|
|
}{
|
|
{"identical", &s3_lifecycle_pb.EntryIdentity{MtimeNs: 100, Size: 2048, HeadFid: "1,abc", ExtendedHash: []byte{0x01, 0x02}}, true},
|
|
{"mtime-drift", &s3_lifecycle_pb.EntryIdentity{MtimeNs: 101, Size: 2048, HeadFid: "1,abc", ExtendedHash: []byte{0x01, 0x02}}, false},
|
|
{"size-drift", &s3_lifecycle_pb.EntryIdentity{MtimeNs: 100, Size: 2049, HeadFid: "1,abc", ExtendedHash: []byte{0x01, 0x02}}, false},
|
|
{"fid-drift", &s3_lifecycle_pb.EntryIdentity{MtimeNs: 100, Size: 2048, HeadFid: "1,xyz", ExtendedHash: []byte{0x01, 0x02}}, false},
|
|
{"extended-drift", &s3_lifecycle_pb.EntryIdentity{MtimeNs: 100, Size: 2048, HeadFid: "1,abc", ExtendedHash: []byte{0x03, 0x04}}, false},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := identityMatches(c.live, base); got != c.want {
|
|
t.Fatalf("want %v, got %v", c.want, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLifecycleDelete_RejectsEmptyRequest(t *testing.T) {
|
|
s := &S3ApiServer{}
|
|
resp, err := s.LifecycleDelete(nil, &s3_lifecycle_pb.LifecycleDeleteRequest{})
|
|
if err != nil {
|
|
t.Fatalf("unexpected gRPC error: %v", err)
|
|
}
|
|
if resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED {
|
|
t.Fatalf("empty request should be BLOCKED, got %v", resp.Outcome)
|
|
}
|
|
}
|
|
|
|
func TestLifecycleAbortMPU_RejectsTraversalUploadIDs(t *testing.T) {
|
|
// "." and ".." pass the no-slash check but resolve to the bucket
|
|
// root via util.JoinPath; they must be rejected before any rm call.
|
|
s := &S3ApiServer{}
|
|
cases := []string{
|
|
"",
|
|
".uploads",
|
|
".uploads/",
|
|
".uploads/.",
|
|
".uploads/..",
|
|
".uploads/u1/extra",
|
|
}
|
|
for _, path := range cases {
|
|
t.Run(path, func(t *testing.T) {
|
|
resp, err := s.LifecycleDelete(nil, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
|
Bucket: "bk",
|
|
ObjectPath: path,
|
|
ActionKind: s3_lifecycle_pb.ActionKind_ABORT_MPU,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected gRPC error: %v", err)
|
|
}
|
|
if resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED {
|
|
t.Fatalf("path %q: outcome=%v reason=%q, want BLOCKED",
|
|
path, resp.Outcome, resp.Reason)
|
|
}
|
|
})
|
|
}
|
|
}
|