mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-11 17:10:40 +02:00
* admin: add bucket lifecycle rule editing * address greptile's comments * more small fixes * coderabbit's comments * more comment fixes * more fixes * more * maybe last * last ? * 14850 * 14851 * filer: stamp the content MD5 on every SaveInsideFiler write An entry's ETag falls back to Attributes.Md5, so conditional writers key IF_ETAG_MATCH off it. SaveInsideFiler carried the looked-up attributes forward without refreshing the hash, leaving it describing whatever the previous writer stored: a later conditional write matched the stale hash and overwrote content that had already changed. * s3api: give the bucket lifecycle constants and the write route key one definition each The extended-attribute keys, the XML size cap and the object-write ring key prefix were each spelled out in two places, so the admin dashboard's copies could drift from the gateway's. Move them to the packages both sides already import and alias them where the short local name reads better. * admin: patch the bucket entry's lifecycle keys instead of rewriting the entry The save read the bucket entry, edited its extended map and wrote the whole entry back, guarded by IF_UNMODIFIED_SINCE. Nothing that writes a bucket entry advances its mtime - not the S3 gateway's patchBucketEntry, not SetBucketOwner, not SetBucketQuota - so the guard never fired and the stale snapshot reverted whatever else had changed since the lookup. Send the PATCH_EXTENDED mutation the S3 gateway already uses for these keys: the filer re-reads and merges under the bucket path lock, so only the two lifecycle keys move. That removes the reason for the mtime snapshot, the verification retry loop and the compensating restore of the cleared day-TTL rules, which the migration now logs instead. * s3api: run the delete-lifecycle day-TTL migration through the shared helper DeleteBucketLifecycleHandler kept its own copy of the read-strip-write sequence the put handler now shares, including a missing return that let a ToText failure persist a truncated filer.conf and write a second response. It also wrote the whole file back unconditionally, reverting any concurrent edit; the shared helper writes conditionally. * admin: answer 404 when a lifecycle request names a bucket that does not exist Every SetBucketLifecycle failure came back as 500, including the lookup miss for an unknown bucket, so a client or monitor read a caller error as a server fault and retried it. * s3api: emit lifecycle XML a client would recognize Two changes to what MarshalCanonical writes, both visible through GetBucketLifecycleConfiguration, which replays the stored bytes verbatim: stamp the S3 namespace on the root, and put a size range under <And>. A <Filter> carries one predicate, so two size bounds side by side is a shape AWS does not document. Parsing still accepts either. * admin: fix the lifecycle editor's handling of stored status, deletes and empty saves Four things the editor got wrong: A stored <Status> the S3 API never validated, say 'enabled', left both radio buttons unchecked, so reading the form threw on a null querySelector result and Save did nothing. Collapse anything but an exact 'Enabled' to 'Disabled', which is what the engine already does with it. Deleting a rule re-rendered an open edit form from the snapshot taken when editing began, discarding what had been typed; every other transition folds the form in first. The Transition warning only matched a bare <Transition>, missing the form with attributes, self-closed or namespace-prefixed. Saving an emptied rule list clears the configuration through a path with no prompt, next to a Delete-all-rules button that asks. Also collapses the three divergent copies of formatBytes on this page to one. * filer: stop the day-TTL migration from deleting an operator's path rule The migration removed every rule under the bucket's path that carried a day TTL in the bucket's collection. The add path it is retiring used AddLocationConf, which merged its TTL onto whatever already sat at the prefix, so a rule can hold operator settings the lifecycle path never wrote - a disk type, WORM retention, a read-only flag, a placement pin. Deleting the whole rule to retire its TTL took those with it, leaving objects under that prefix on defaults nobody asked for. Delete only rules shaped like ones the add path created from scratch; anything else keeps its settings and loses just the TTL. --------- Co-authored-by: Chris Lu <chris.lu@gmail.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
592 lines
22 KiB
Go
592 lines
22 KiB
Go
package filer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
func TestFilerConf(t *testing.T) {
|
|
|
|
fc := NewFilerConf()
|
|
|
|
conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{
|
|
{
|
|
LocationPrefix: "/buckets/abc",
|
|
Collection: "abc",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/abcd",
|
|
Collection: "abcd",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/",
|
|
Replication: "001",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/xxx",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/xxx/yyy",
|
|
ReadOnly: false,
|
|
},
|
|
}}
|
|
fc.doLoadConf(conf)
|
|
|
|
assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection)
|
|
assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection)
|
|
assert.Equal(t, "001", fc.MatchStorageRule("/buckets/abc/jasdf").Replication)
|
|
|
|
assert.Equal(t, true, fc.MatchStorageRule("/buckets/xxx/yyy/zzz").ReadOnly)
|
|
assert.Equal(t, false, fc.MatchStorageRule("/buckets/other").ReadOnly)
|
|
|
|
}
|
|
|
|
func TestWormInheritance(t *testing.T) {
|
|
fc := NewFilerConf()
|
|
fc.doLoadConf(&filer_pb.FilerConf{
|
|
Version: FilerConfVersion,
|
|
Locations: []*filer_pb.FilerConf_PathConf{
|
|
{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true), Ttl: "7d"},
|
|
{LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"},
|
|
{LocationPrefix: "/buckets/b/scratch/", Worm: proto.Bool(false)},
|
|
{LocationPrefix: "/buckets/b/scratch/keep/", Worm: proto.Bool(true)},
|
|
},
|
|
})
|
|
|
|
// a rule that says nothing about worm keeps inheriting it, along with the ttl
|
|
rule := fc.MatchStorageRule("/buckets/b/quiet/x")
|
|
assert.True(t, rule.GetWorm())
|
|
assert.Equal(t, "7d", rule.Ttl)
|
|
|
|
// an explicit false turns it off, and a deeper rule turns it back on
|
|
assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetWorm())
|
|
assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetWorm())
|
|
|
|
// paths with no rule at all are unaffected
|
|
assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetWorm())
|
|
}
|
|
|
|
// TestWormLegacyFalseIsNotAnOverride pins that the explicit "worm": false every
|
|
// version 0 configuration carries does not read as a per-path opt-out.
|
|
func TestWormLegacyFalseIsNotAnOverride(t *testing.T) {
|
|
const conf = `{
|
|
"locations": [
|
|
{"locationPrefix": "/buckets/b/", "worm": true},
|
|
{"locationPrefix": "/buckets/b/sub/", "collection": "sub", "worm": false}
|
|
]
|
|
}`
|
|
|
|
fc := NewFilerConf()
|
|
assert.NoError(t, fc.LoadFromBytes([]byte(conf)))
|
|
assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
|
|
// the same file at the current version means what it says
|
|
fc = NewFilerConf()
|
|
assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 1,`+conf[1:])))
|
|
assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
}
|
|
|
|
// TestWormSurvivesRoundTrip guards the write side: an explicit false has to be
|
|
// stamped along with a version that says to honor it.
|
|
func TestWormSurvivesRoundTrip(t *testing.T) {
|
|
fc := NewFilerConf()
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true)})
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Worm: proto.Bool(false)})
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"})
|
|
|
|
var buf bytes.Buffer
|
|
assert.NoError(t, fc.ToText(&buf))
|
|
|
|
reloaded := NewFilerConf()
|
|
assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes()))
|
|
assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm())
|
|
}
|
|
|
|
// TestClonePathConf verifies that ClonePathConf copies all exported fields.
|
|
// Uses reflection to automatically detect new fields added to the protobuf,
|
|
// ensuring the test fails if ClonePathConf is not updated for new fields.
|
|
func TestClonePathConf(t *testing.T) {
|
|
// Create a fully-populated PathConf with non-zero values for all fields
|
|
src := &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/test/path",
|
|
Collection: "test_collection",
|
|
Replication: "001",
|
|
Ttl: "7d",
|
|
DiskType: "ssd",
|
|
Fsync: true,
|
|
VolumeGrowthCount: 5,
|
|
ReadOnly: true,
|
|
MaxFileNameLength: 255,
|
|
DataCenter: "dc1",
|
|
Rack: "rack1",
|
|
DataNode: "node1",
|
|
DisableChunkDeletion: true,
|
|
Worm: proto.Bool(true),
|
|
WormGracePeriodSeconds: 3600,
|
|
WormRetentionTimeSeconds: 86400,
|
|
}
|
|
|
|
clone := ClonePathConf(src)
|
|
|
|
// Verify it's a different object
|
|
assert.NotSame(t, src, clone, "ClonePathConf should return a new object, not the same pointer")
|
|
|
|
// Use reflection to compare all exported fields
|
|
// This will automatically catch any new fields added to the protobuf
|
|
srcVal := reflect.ValueOf(src).Elem()
|
|
cloneVal := reflect.ValueOf(clone).Elem()
|
|
srcType := srcVal.Type()
|
|
|
|
for i := 0; i < srcType.NumField(); i++ {
|
|
field := srcType.Field(i)
|
|
|
|
// Skip unexported fields (protobuf internal fields like sizeCache, unknownFields)
|
|
if !field.IsExported() {
|
|
continue
|
|
}
|
|
|
|
srcField := srcVal.Field(i)
|
|
cloneField := cloneVal.Field(i)
|
|
|
|
// Compare field values
|
|
if !reflect.DeepEqual(srcField.Interface(), cloneField.Interface()) {
|
|
t.Errorf("Field %s not copied correctly: src=%v, clone=%v",
|
|
field.Name, srcField.Interface(), cloneField.Interface())
|
|
}
|
|
}
|
|
|
|
// Additionally verify that all exported fields in src are non-zero
|
|
// This ensures we're testing with fully populated data
|
|
for i := 0; i < srcType.NumField(); i++ {
|
|
field := srcType.Field(i)
|
|
if !field.IsExported() {
|
|
continue
|
|
}
|
|
|
|
srcField := srcVal.Field(i)
|
|
if srcField.IsZero() {
|
|
t.Errorf("Test setup error: field %s has zero value, update test to set a non-zero value", field.Name)
|
|
}
|
|
}
|
|
|
|
// Verify mutation of clone doesn't affect source
|
|
clone.Collection = "modified"
|
|
clone.ReadOnly = false
|
|
*clone.Worm = false
|
|
assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection")
|
|
assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly")
|
|
assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm")
|
|
}
|
|
|
|
func TestClonePathConfNil(t *testing.T) {
|
|
clone := ClonePathConf(nil)
|
|
assert.NotNil(t, clone, "ClonePathConf(nil) should return a non-nil empty PathConf")
|
|
assert.Equal(t, "", clone.LocationPrefix, "ClonePathConf(nil) should return empty PathConf")
|
|
}
|
|
|
|
func TestApplyBucketQuotaReadOnly(t *testing.T) {
|
|
const prefix = "/buckets/b/"
|
|
|
|
// over quota: flips to read-only
|
|
fc := NewFilerConf()
|
|
readOnly, changed := fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.True(t, changed)
|
|
assert.True(t, readOnly)
|
|
assert.True(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
|
|
// still over quota: no change
|
|
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.False(t, changed)
|
|
|
|
// back under quota: flips to writable
|
|
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
|
|
assert.True(t, changed)
|
|
assert.False(t, readOnly)
|
|
assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
|
|
// quota disabled leaves the flag untouched, so manual locks survive
|
|
fc = NewFilerConf()
|
|
fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, -1)
|
|
assert.False(t, changed)
|
|
assert.True(t, readOnly)
|
|
|
|
// under quota and not read-only: no rule churn
|
|
fc = NewFilerConf()
|
|
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
|
|
assert.False(t, changed)
|
|
}
|
|
|
|
func TestClearReadOnly(t *testing.T) {
|
|
const prefix = "/buckets/b/"
|
|
|
|
fc := NewFilerConf()
|
|
assert.False(t, fc.ClearReadOnly(prefix), "no rule to clear")
|
|
|
|
// locked by quota enforcement, then quota removed: still clearable
|
|
fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.True(t, fc.ClearReadOnly(prefix))
|
|
assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
assert.False(t, fc.ClearReadOnly(prefix), "already writable")
|
|
|
|
// clearing the flag keeps the rule's other settings
|
|
fc = NewFilerConf()
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: prefix, Ttl: "7d", ReadOnly: true})
|
|
assert.True(t, fc.ClearReadOnly(prefix))
|
|
rule := fc.MatchStorageRule(prefix)
|
|
assert.False(t, rule.ReadOnly)
|
|
assert.Equal(t, "7d", rule.Ttl)
|
|
}
|
|
|
|
// fakeFilerConfClient is a minimal in-memory filer_pb.SeaweedFilerClient that
|
|
// only supports the single-file round trip ReadInsideFiler/SaveInsideFiler
|
|
// need: lookup, create-if-absent, update. Embedding the interface satisfies
|
|
// the rest of it; calling any other method panics on the nil embedded value.
|
|
type fakeFilerConfClient struct {
|
|
filer_pb.SeaweedFilerClient
|
|
|
|
mu sync.Mutex
|
|
entries map[string]*filer_pb.Entry // key: dir+"/"+name
|
|
}
|
|
|
|
func newFakeFilerConfClient() *fakeFilerConfClient {
|
|
return &fakeFilerConfClient{entries: make(map[string]*filer_pb.Entry)}
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) key(dir, name string) string { return dir + "/" + name }
|
|
|
|
func (c *fakeFilerConfClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
e, ok := c.entries[c.key(in.Directory, in.Name)]
|
|
if !ok {
|
|
return nil, filer_pb.ErrNotFound
|
|
}
|
|
// A real gRPC round trip always hands back an independent copy (proto
|
|
// marshal/unmarshal), so mutating what the caller gets back (as
|
|
// saveFilerConfConditionally does before re-sending it) must not affect
|
|
// what conditionHoldsLocked below compares against.
|
|
return &filer_pb.LookupDirectoryEntryResponse{Entry: cloneFakeEntry(e)}, nil
|
|
}
|
|
|
|
func cloneFakeEntry(e *filer_pb.Entry) *filer_pb.Entry {
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
// proto.Clone rather than a struct copy: filer_pb.Entry embeds
|
|
// protoimpl.MessageState (a sync.Mutex), which a plain `*e` copy would
|
|
// duplicate by value — exactly what go vet's copylocks check flags.
|
|
return proto.Clone(e).(*filer_pb.Entry)
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) CreateEntry(_ context.Context, in *filer_pb.CreateEntryRequest, _ ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
key := c.key(in.Directory, in.Entry.Name)
|
|
if !conditionHoldsLocked(in.Condition, c.entries[key]) {
|
|
return nil, errors.New("precondition failed")
|
|
}
|
|
c.entries[key] = in.Entry
|
|
return &filer_pb.CreateEntryResponse{}, nil
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
key := c.key(in.Directory, in.Entry.Name)
|
|
if !conditionHoldsLocked(in.Condition, c.entries[key]) {
|
|
return nil, errors.New("precondition failed")
|
|
}
|
|
c.entries[key] = in.Entry
|
|
return &filer_pb.UpdateEntryResponse{}, nil
|
|
}
|
|
|
|
// conditionHoldsLocked is a minimal stand-in for the filer server's
|
|
// writeConditionSatisfied (weed/server/filer_grpc_server_condition.go),
|
|
// covering just the clause kinds saveFilerConfConditionally uses, so tests
|
|
// can exercise the CAS path without a real filer server.
|
|
func conditionHoldsLocked(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool {
|
|
if cond == nil {
|
|
return true
|
|
}
|
|
for _, clause := range cond.Clauses {
|
|
switch clause.Kind {
|
|
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
|
if current != nil {
|
|
return false
|
|
}
|
|
case filer_pb.WriteCondition_IF_UNMODIFIED_SINCE:
|
|
if current != nil && current.Attributes != nil && current.Attributes.Mtime > clause.UnixTime {
|
|
return false
|
|
}
|
|
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
|
if current == nil {
|
|
return false
|
|
}
|
|
stored := ""
|
|
if current.Attributes != nil {
|
|
stored = fmt.Sprintf("%x", current.Attributes.Md5)
|
|
}
|
|
matched := false
|
|
for _, want := range clause.Etags {
|
|
if want == stored {
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// putFilerConf seeds the fake client's filer.conf with the given rules.
|
|
func putFilerConf(t *testing.T, client *fakeFilerConfClient, rules ...*filer_pb.FilerConf_PathConf) {
|
|
t.Helper()
|
|
fc := NewFilerConf()
|
|
for _, r := range rules {
|
|
require.NoError(t, fc.SetLocationConf(r))
|
|
}
|
|
var buf bytes.Buffer
|
|
require.NoError(t, fc.ToText(&buf))
|
|
client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)] = &filer_pb.Entry{
|
|
Name: FilerConfName,
|
|
Content: buf.Bytes(),
|
|
Attributes: &filer_pb.FuseAttributes{},
|
|
}
|
|
}
|
|
|
|
// readFilerConfText returns the current filer.conf content stored in the fake
|
|
// client, or "" if none exists yet.
|
|
func readFilerConfText(client *fakeFilerConfClient) string {
|
|
e, ok := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return string(e.Content)
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_NoFilerConf(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
assert.Empty(t, readFilerConfText(client))
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_NoMatchingRule(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/other/",
|
|
Collection: "other",
|
|
Ttl: "7d",
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
assert.Contains(t, readFilerConfText(client), "other")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_RemovesDayTTLUnderBucket(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/",
|
|
Collection: "mybucket",
|
|
Ttl: "7d",
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.False(t, found, "day-TTL rule should have been removed")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsNonDayTTL(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/",
|
|
Collection: "mybucket",
|
|
Ttl: "7m", // minutes, not days: not a legacy lifecycle TTL rule
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.True(t, found, "non-day TTL rule should be left alone")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsOperatorSettingsAndClearsOnlyTheTTL(t *testing.T) {
|
|
// The removed add path merged its TTL onto whatever already sat at the
|
|
// prefix, so a rule carrying operator settings must survive with only its
|
|
// TTL retired - deleting it outright would drop the disk type and the
|
|
// read-only flag with it.
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/logs/",
|
|
Collection: "mybucket",
|
|
Ttl: "30d",
|
|
DiskType: "hdd",
|
|
ReadOnly: true,
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
rule, found := reloaded.GetLocationConf("/buckets/mybucket/logs/")
|
|
require.True(t, found, "a rule carrying operator settings must not be deleted")
|
|
assert.Equal(t, "", rule.Ttl, "the legacy day TTL should be gone")
|
|
assert.Equal(t, "hdd", rule.DiskType, "the operator's disk type must survive")
|
|
assert.True(t, rule.ReadOnly, "the operator's read-only flag must survive")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsWormSettings(t *testing.T) {
|
|
worm := true
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/locked/",
|
|
Collection: "mybucket",
|
|
Ttl: "7d",
|
|
Worm: &worm,
|
|
WormRetentionTimeSeconds: 3600,
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
rule, found := reloaded.GetLocationConf("/buckets/mybucket/locked/")
|
|
require.True(t, found, "a WORM rule must not be deleted to retire a TTL")
|
|
assert.Equal(t, "", rule.Ttl)
|
|
assert.True(t, rule.GetWorm())
|
|
assert.Equal(t, uint64(3600), rule.WormRetentionTimeSeconds)
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsOtherBucketsAndCollections(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client,
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/", Collection: "mybucket", Ttl: "7d"},
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "7d"},
|
|
// nested under the target bucket's path but tagged to a different
|
|
// collection (e.g. a filer-group-prefixed name): must not be swept up
|
|
// by this bucket's cleanup, since GetCollectionTtls filters by collection.
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/nested/", Collection: "group_mybucket", Ttl: "7d"},
|
|
)
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.False(t, found, "the target bucket's day-TTL rule should be gone")
|
|
|
|
text := readFilerConfText(client)
|
|
assert.Contains(t, text, "other")
|
|
assert.Contains(t, text, "group_mybucket")
|
|
}
|
|
|
|
// bumpMtime models a writer that landed after unmodifiedSince, so an
|
|
// IF_UNMODIFIED_SINCE precondition built on that timestamp no longer holds.
|
|
func bumpMtime(client *fakeFilerConfClient, unmodifiedSince int64) {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
e := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
|
|
if e == nil {
|
|
return
|
|
}
|
|
if e.Attributes == nil {
|
|
e.Attributes = &filer_pb.FuseAttributes{}
|
|
}
|
|
e.Attributes.Mtime = unmodifiedSince + 1
|
|
}
|
|
|
|
func TestSaveFilerConfConditionally_RejectsUpdateModifiedConcurrently(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
|
|
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, snap.entry)
|
|
require.Empty(t, snap.entry.Attributes.Md5, "test setup should model a pre-existing filer.conf with no stamped Md5")
|
|
|
|
// Someone else writes filer.conf after our read but before our write.
|
|
bumpMtime(client, snap.entry.Attributes.Mtime)
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap)
|
|
assert.Error(t, err, "expected a concurrent modification to fail the conditional write")
|
|
}
|
|
|
|
// TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification
|
|
// exercises the primary path: once an entry has been written through this
|
|
// function (so it carries a content Md5), a later write with a stale
|
|
// snapshot is rejected by the exact IF_ETAG_MATCH check even when the
|
|
// concurrent writer happened to leave mtime unchanged — something a
|
|
// mtime-only precondition would have missed.
|
|
func TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
|
|
|
|
// First write through saveFilerConfConditionally stamps Md5, moving
|
|
// subsequent reads onto the exact-content check.
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NoError(t, saveFilerConfConditionally(context.Background(), client, snap))
|
|
|
|
snap2, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, snap2.entry.Attributes.Md5, "expected Md5 to have been stamped by the prior write")
|
|
|
|
// A concurrent writer goes through SaveInsideFiler, the path every
|
|
// other filer.conf writer uses. It lands in the same wall-clock second,
|
|
// so only the content hash can catch it.
|
|
concurrentFc := NewFilerConf()
|
|
require.NoError(t, concurrentFc.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
require.NoError(t, concurrentFc.SetLocationConf(&filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "3d",
|
|
}))
|
|
var buf bytes.Buffer
|
|
require.NoError(t, concurrentFc.ToText(&buf))
|
|
require.NoError(t, SaveInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName, buf.Bytes()))
|
|
client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)].Attributes.Mtime = snap2.entry.Attributes.Mtime
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap2)
|
|
assert.Error(t, err, "expected the exact-content check to reject a write whose baseline content hash no longer matches")
|
|
assert.Contains(t, readFilerConfText(client), "/buckets/other/", "the concurrent writer's content must survive")
|
|
}
|
|
|
|
func TestSaveFilerConfConditionally_RejectsCreateWhenCreatedConcurrently(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.Nil(t, snap.entry, "filer.conf should not exist yet")
|
|
|
|
// Someone else creates filer.conf after our read but before our write.
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Collection: "b", Ttl: "3d"})
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap)
|
|
assert.Error(t, err, "expected a concurrent create to fail the conditional write")
|
|
assert.Contains(t, readFilerConfText(client), "/buckets/b/", "the concurrent writer's content must survive")
|
|
}
|