mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-07 06:47:51 +02:00
* filer: skip UpdateEntry when inline content is unchanged SaveInsideFiler rewrites config files (IAM identities, filer.conf, remote mappings, policies) unconditionally. Each no-op UpdateEntry is a metadata event the local meta log persists to /topics/.system/log, which appends a chunk to a volume. A client that rewrites identical config on a timer, e.g. the seaweedfs-operator 5-minute resync calling UpdateUser with unchanged actions, keeps .dat/.idx files growing on an otherwise idle cluster and prevents HDD spindown (seaweedfs/seaweedfs#11571). Skip the UpdateEntry when the stored inline content is byte-identical, so unchanged writes produce no metadata event and no volume writes. * filer: test that identical SaveInsideFiler writes skip UpdateEntry * filer: require stamped Md5 before skipping identical writes An entry holding identical content but no Md5 (written before hashing, or by a tool that cleared it) would never get the stamp that IF_ETAG_MATCH conditional writes key off. Skip only when both the stored content and its Md5 match, so one write still lands to repair the stamp.
594 lines
22 KiB
Go
594 lines
22 KiB
Go
package filer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
func TestFilerConf(t *testing.T) {
|
|
|
|
fc := NewFilerConf()
|
|
|
|
conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{
|
|
{
|
|
LocationPrefix: "/buckets/abc",
|
|
Collection: "abc",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/abcd",
|
|
Collection: "abcd",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/",
|
|
Replication: "001",
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets",
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/xxx",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
LocationPrefix: "/buckets/xxx/yyy",
|
|
ReadOnly: false,
|
|
},
|
|
}}
|
|
fc.doLoadConf(conf)
|
|
|
|
assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection)
|
|
assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection)
|
|
assert.Equal(t, "001", fc.MatchStorageRule("/buckets/abc/jasdf").Replication)
|
|
|
|
assert.Equal(t, true, fc.MatchStorageRule("/buckets/xxx/yyy/zzz").ReadOnly)
|
|
assert.Equal(t, false, fc.MatchStorageRule("/buckets/other").ReadOnly)
|
|
|
|
}
|
|
|
|
func TestWormInheritance(t *testing.T) {
|
|
fc := NewFilerConf()
|
|
fc.doLoadConf(&filer_pb.FilerConf{
|
|
Version: FilerConfVersion,
|
|
Locations: []*filer_pb.FilerConf_PathConf{
|
|
{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true), Ttl: "7d"},
|
|
{LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"},
|
|
{LocationPrefix: "/buckets/b/scratch/", Worm: proto.Bool(false)},
|
|
{LocationPrefix: "/buckets/b/scratch/keep/", Worm: proto.Bool(true)},
|
|
},
|
|
})
|
|
|
|
// a rule that says nothing about worm keeps inheriting it, along with the ttl
|
|
rule := fc.MatchStorageRule("/buckets/b/quiet/x")
|
|
assert.True(t, rule.GetWorm())
|
|
assert.Equal(t, "7d", rule.Ttl)
|
|
|
|
// an explicit false turns it off, and a deeper rule turns it back on
|
|
assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetWorm())
|
|
assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetWorm())
|
|
|
|
// paths with no rule at all are unaffected
|
|
assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetWorm())
|
|
}
|
|
|
|
// TestWormLegacyFalseIsNotAnOverride pins that the explicit "worm": false every
|
|
// version 0 configuration carries does not read as a per-path opt-out.
|
|
func TestWormLegacyFalseIsNotAnOverride(t *testing.T) {
|
|
const conf = `{
|
|
"locations": [
|
|
{"locationPrefix": "/buckets/b/", "worm": true},
|
|
{"locationPrefix": "/buckets/b/sub/", "collection": "sub", "worm": false}
|
|
]
|
|
}`
|
|
|
|
fc := NewFilerConf()
|
|
assert.NoError(t, fc.LoadFromBytes([]byte(conf)))
|
|
assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
|
|
// the same file at the current version means what it says
|
|
fc = NewFilerConf()
|
|
assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 1,`+conf[1:])))
|
|
assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
}
|
|
|
|
// TestWormSurvivesRoundTrip guards the write side: an explicit false has to be
|
|
// stamped along with a version that says to honor it.
|
|
func TestWormSurvivesRoundTrip(t *testing.T) {
|
|
fc := NewFilerConf()
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true)})
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Worm: proto.Bool(false)})
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"})
|
|
|
|
var buf bytes.Buffer
|
|
assert.NoError(t, fc.ToText(&buf))
|
|
|
|
reloaded := NewFilerConf()
|
|
assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes()))
|
|
assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
|
assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm())
|
|
}
|
|
|
|
// TestClonePathConf verifies that ClonePathConf copies all exported fields.
|
|
// Uses reflection to automatically detect new fields added to the protobuf,
|
|
// ensuring the test fails if ClonePathConf is not updated for new fields.
|
|
func TestClonePathConf(t *testing.T) {
|
|
// Create a fully-populated PathConf with non-zero values for all fields
|
|
src := &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/test/path",
|
|
Collection: "test_collection",
|
|
Replication: "001",
|
|
Ttl: "7d",
|
|
DiskType: "ssd",
|
|
Fsync: true,
|
|
VolumeGrowthCount: 5,
|
|
ReadOnly: true,
|
|
MaxFileNameLength: 255,
|
|
DataCenter: "dc1",
|
|
Rack: "rack1",
|
|
DataNode: "node1",
|
|
DisableChunkDeletion: true,
|
|
Worm: proto.Bool(true),
|
|
WormGracePeriodSeconds: 3600,
|
|
WormRetentionTimeSeconds: 86400,
|
|
}
|
|
|
|
clone := ClonePathConf(src)
|
|
|
|
// Verify it's a different object
|
|
assert.NotSame(t, src, clone, "ClonePathConf should return a new object, not the same pointer")
|
|
|
|
// Use reflection to compare all exported fields
|
|
// This will automatically catch any new fields added to the protobuf
|
|
srcVal := reflect.ValueOf(src).Elem()
|
|
cloneVal := reflect.ValueOf(clone).Elem()
|
|
srcType := srcVal.Type()
|
|
|
|
for i := 0; i < srcType.NumField(); i++ {
|
|
field := srcType.Field(i)
|
|
|
|
// Skip unexported fields (protobuf internal fields like sizeCache, unknownFields)
|
|
if !field.IsExported() {
|
|
continue
|
|
}
|
|
|
|
srcField := srcVal.Field(i)
|
|
cloneField := cloneVal.Field(i)
|
|
|
|
// Compare field values
|
|
if !reflect.DeepEqual(srcField.Interface(), cloneField.Interface()) {
|
|
t.Errorf("Field %s not copied correctly: src=%v, clone=%v",
|
|
field.Name, srcField.Interface(), cloneField.Interface())
|
|
}
|
|
}
|
|
|
|
// Additionally verify that all exported fields in src are non-zero
|
|
// This ensures we're testing with fully populated data
|
|
for i := 0; i < srcType.NumField(); i++ {
|
|
field := srcType.Field(i)
|
|
if !field.IsExported() {
|
|
continue
|
|
}
|
|
|
|
srcField := srcVal.Field(i)
|
|
if srcField.IsZero() {
|
|
t.Errorf("Test setup error: field %s has zero value, update test to set a non-zero value", field.Name)
|
|
}
|
|
}
|
|
|
|
// Verify mutation of clone doesn't affect source
|
|
clone.Collection = "modified"
|
|
clone.ReadOnly = false
|
|
*clone.Worm = false
|
|
assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection")
|
|
assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly")
|
|
assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm")
|
|
}
|
|
|
|
func TestClonePathConfNil(t *testing.T) {
|
|
clone := ClonePathConf(nil)
|
|
assert.NotNil(t, clone, "ClonePathConf(nil) should return a non-nil empty PathConf")
|
|
assert.Equal(t, "", clone.LocationPrefix, "ClonePathConf(nil) should return empty PathConf")
|
|
}
|
|
|
|
func TestApplyBucketQuotaReadOnly(t *testing.T) {
|
|
const prefix = "/buckets/b/"
|
|
|
|
// over quota: flips to read-only
|
|
fc := NewFilerConf()
|
|
readOnly, changed := fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.True(t, changed)
|
|
assert.True(t, readOnly)
|
|
assert.True(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
|
|
// still over quota: no change
|
|
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.False(t, changed)
|
|
|
|
// back under quota: flips to writable
|
|
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
|
|
assert.True(t, changed)
|
|
assert.False(t, readOnly)
|
|
assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
|
|
// quota disabled leaves the flag untouched, so manual locks survive
|
|
fc = NewFilerConf()
|
|
fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, -1)
|
|
assert.False(t, changed)
|
|
assert.True(t, readOnly)
|
|
|
|
// under quota and not read-only: no rule churn
|
|
fc = NewFilerConf()
|
|
_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
|
|
assert.False(t, changed)
|
|
}
|
|
|
|
func TestClearReadOnly(t *testing.T) {
|
|
const prefix = "/buckets/b/"
|
|
|
|
fc := NewFilerConf()
|
|
assert.False(t, fc.ClearReadOnly(prefix), "no rule to clear")
|
|
|
|
// locked by quota enforcement, then quota removed: still clearable
|
|
fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
|
|
assert.True(t, fc.ClearReadOnly(prefix))
|
|
assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
|
|
assert.False(t, fc.ClearReadOnly(prefix), "already writable")
|
|
|
|
// clearing the flag keeps the rule's other settings
|
|
fc = NewFilerConf()
|
|
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: prefix, Ttl: "7d", ReadOnly: true})
|
|
assert.True(t, fc.ClearReadOnly(prefix))
|
|
rule := fc.MatchStorageRule(prefix)
|
|
assert.False(t, rule.ReadOnly)
|
|
assert.Equal(t, "7d", rule.Ttl)
|
|
}
|
|
|
|
// fakeFilerConfClient is a minimal in-memory filer_pb.SeaweedFilerClient that
|
|
// only supports the single-file round trip ReadInsideFiler/SaveInsideFiler
|
|
// need: lookup, create-if-absent, update. Embedding the interface satisfies
|
|
// the rest of it; calling any other method panics on the nil embedded value.
|
|
type fakeFilerConfClient struct {
|
|
filer_pb.SeaweedFilerClient
|
|
|
|
mu sync.Mutex
|
|
entries map[string]*filer_pb.Entry // key: dir+"/"+name
|
|
updateCalls int
|
|
}
|
|
|
|
func newFakeFilerConfClient() *fakeFilerConfClient {
|
|
return &fakeFilerConfClient{entries: make(map[string]*filer_pb.Entry)}
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) key(dir, name string) string { return dir + "/" + name }
|
|
|
|
func (c *fakeFilerConfClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
e, ok := c.entries[c.key(in.Directory, in.Name)]
|
|
if !ok {
|
|
return nil, filer_pb.ErrNotFound
|
|
}
|
|
// A real gRPC round trip always hands back an independent copy (proto
|
|
// marshal/unmarshal), so mutating what the caller gets back (as
|
|
// saveFilerConfConditionally does before re-sending it) must not affect
|
|
// what conditionHoldsLocked below compares against.
|
|
return &filer_pb.LookupDirectoryEntryResponse{Entry: cloneFakeEntry(e)}, nil
|
|
}
|
|
|
|
func cloneFakeEntry(e *filer_pb.Entry) *filer_pb.Entry {
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
// proto.Clone rather than a struct copy: filer_pb.Entry embeds
|
|
// protoimpl.MessageState (a sync.Mutex), which a plain `*e` copy would
|
|
// duplicate by value — exactly what go vet's copylocks check flags.
|
|
return proto.Clone(e).(*filer_pb.Entry)
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) CreateEntry(_ context.Context, in *filer_pb.CreateEntryRequest, _ ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
key := c.key(in.Directory, in.Entry.Name)
|
|
if !conditionHoldsLocked(in.Condition, c.entries[key]) {
|
|
return nil, errors.New("precondition failed")
|
|
}
|
|
c.entries[key] = in.Entry
|
|
return &filer_pb.CreateEntryResponse{}, nil
|
|
}
|
|
|
|
func (c *fakeFilerConfClient) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.updateCalls++
|
|
key := c.key(in.Directory, in.Entry.Name)
|
|
if !conditionHoldsLocked(in.Condition, c.entries[key]) {
|
|
return nil, errors.New("precondition failed")
|
|
}
|
|
c.entries[key] = in.Entry
|
|
return &filer_pb.UpdateEntryResponse{}, nil
|
|
}
|
|
|
|
// conditionHoldsLocked is a minimal stand-in for the filer server's
|
|
// writeConditionSatisfied (weed/server/filer_grpc_server_condition.go),
|
|
// covering just the clause kinds saveFilerConfConditionally uses, so tests
|
|
// can exercise the CAS path without a real filer server.
|
|
func conditionHoldsLocked(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool {
|
|
if cond == nil {
|
|
return true
|
|
}
|
|
for _, clause := range cond.Clauses {
|
|
switch clause.Kind {
|
|
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
|
if current != nil {
|
|
return false
|
|
}
|
|
case filer_pb.WriteCondition_IF_UNMODIFIED_SINCE:
|
|
if current != nil && current.Attributes != nil && current.Attributes.Mtime > clause.UnixTime {
|
|
return false
|
|
}
|
|
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
|
if current == nil {
|
|
return false
|
|
}
|
|
stored := ""
|
|
if current.Attributes != nil {
|
|
stored = fmt.Sprintf("%x", current.Attributes.Md5)
|
|
}
|
|
matched := false
|
|
for _, want := range clause.Etags {
|
|
if want == stored {
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// putFilerConf seeds the fake client's filer.conf with the given rules.
|
|
func putFilerConf(t *testing.T, client *fakeFilerConfClient, rules ...*filer_pb.FilerConf_PathConf) {
|
|
t.Helper()
|
|
fc := NewFilerConf()
|
|
for _, r := range rules {
|
|
require.NoError(t, fc.SetLocationConf(r))
|
|
}
|
|
var buf bytes.Buffer
|
|
require.NoError(t, fc.ToText(&buf))
|
|
client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)] = &filer_pb.Entry{
|
|
Name: FilerConfName,
|
|
Content: buf.Bytes(),
|
|
Attributes: &filer_pb.FuseAttributes{},
|
|
}
|
|
}
|
|
|
|
// readFilerConfText returns the current filer.conf content stored in the fake
|
|
// client, or "" if none exists yet.
|
|
func readFilerConfText(client *fakeFilerConfClient) string {
|
|
e, ok := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return string(e.Content)
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_NoFilerConf(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
assert.Empty(t, readFilerConfText(client))
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_NoMatchingRule(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/other/",
|
|
Collection: "other",
|
|
Ttl: "7d",
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
assert.Contains(t, readFilerConfText(client), "other")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_RemovesDayTTLUnderBucket(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/",
|
|
Collection: "mybucket",
|
|
Ttl: "7d",
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.False(t, found, "day-TTL rule should have been removed")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsNonDayTTL(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/",
|
|
Collection: "mybucket",
|
|
Ttl: "7m", // minutes, not days: not a legacy lifecycle TTL rule
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.True(t, found, "non-day TTL rule should be left alone")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsOperatorSettingsAndClearsOnlyTheTTL(t *testing.T) {
|
|
// The removed add path merged its TTL onto whatever already sat at the
|
|
// prefix, so a rule carrying operator settings must survive with only its
|
|
// TTL retired - deleting it outright would drop the disk type and the
|
|
// read-only flag with it.
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/logs/",
|
|
Collection: "mybucket",
|
|
Ttl: "30d",
|
|
DiskType: "hdd",
|
|
ReadOnly: true,
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
rule, found := reloaded.GetLocationConf("/buckets/mybucket/logs/")
|
|
require.True(t, found, "a rule carrying operator settings must not be deleted")
|
|
assert.Equal(t, "", rule.Ttl, "the legacy day TTL should be gone")
|
|
assert.Equal(t, "hdd", rule.DiskType, "the operator's disk type must survive")
|
|
assert.True(t, rule.ReadOnly, "the operator's read-only flag must survive")
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsWormSettings(t *testing.T) {
|
|
worm := true
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/mybucket/locked/",
|
|
Collection: "mybucket",
|
|
Ttl: "7d",
|
|
Worm: &worm,
|
|
WormRetentionTimeSeconds: 3600,
|
|
})
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
rule, found := reloaded.GetLocationConf("/buckets/mybucket/locked/")
|
|
require.True(t, found, "a WORM rule must not be deleted to retire a TTL")
|
|
assert.Equal(t, "", rule.Ttl)
|
|
assert.True(t, rule.GetWorm())
|
|
assert.Equal(t, uint64(3600), rule.WormRetentionTimeSeconds)
|
|
}
|
|
|
|
func TestClearBucketLifecycleDayTTLs_KeepsOtherBucketsAndCollections(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client,
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/", Collection: "mybucket", Ttl: "7d"},
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "7d"},
|
|
// nested under the target bucket's path but tagged to a different
|
|
// collection (e.g. a filer-group-prefixed name): must not be swept up
|
|
// by this bucket's cleanup, since GetCollectionTtls filters by collection.
|
|
&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/nested/", Collection: "group_mybucket", Ttl: "7d"},
|
|
)
|
|
|
|
require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
|
|
|
|
reloaded := NewFilerConf()
|
|
require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
_, found := reloaded.GetLocationConf("/buckets/mybucket/")
|
|
assert.False(t, found, "the target bucket's day-TTL rule should be gone")
|
|
|
|
text := readFilerConfText(client)
|
|
assert.Contains(t, text, "other")
|
|
assert.Contains(t, text, "group_mybucket")
|
|
}
|
|
|
|
// bumpMtime models a writer that landed after unmodifiedSince, so an
|
|
// IF_UNMODIFIED_SINCE precondition built on that timestamp no longer holds.
|
|
func bumpMtime(client *fakeFilerConfClient, unmodifiedSince int64) {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
e := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
|
|
if e == nil {
|
|
return
|
|
}
|
|
if e.Attributes == nil {
|
|
e.Attributes = &filer_pb.FuseAttributes{}
|
|
}
|
|
e.Attributes.Mtime = unmodifiedSince + 1
|
|
}
|
|
|
|
func TestSaveFilerConfConditionally_RejectsUpdateModifiedConcurrently(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
|
|
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, snap.entry)
|
|
require.Empty(t, snap.entry.Attributes.Md5, "test setup should model a pre-existing filer.conf with no stamped Md5")
|
|
|
|
// Someone else writes filer.conf after our read but before our write.
|
|
bumpMtime(client, snap.entry.Attributes.Mtime)
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap)
|
|
assert.Error(t, err, "expected a concurrent modification to fail the conditional write")
|
|
}
|
|
|
|
// TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification
|
|
// exercises the primary path: once an entry has been written through this
|
|
// function (so it carries a content Md5), a later write with a stale
|
|
// snapshot is rejected by the exact IF_ETAG_MATCH check even when the
|
|
// concurrent writer happened to leave mtime unchanged — something a
|
|
// mtime-only precondition would have missed.
|
|
func TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
|
|
|
|
// First write through saveFilerConfConditionally stamps Md5, moving
|
|
// subsequent reads onto the exact-content check.
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NoError(t, saveFilerConfConditionally(context.Background(), client, snap))
|
|
|
|
snap2, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, snap2.entry.Attributes.Md5, "expected Md5 to have been stamped by the prior write")
|
|
|
|
// A concurrent writer goes through SaveInsideFiler, the path every
|
|
// other filer.conf writer uses. It lands in the same wall-clock second,
|
|
// so only the content hash can catch it.
|
|
concurrentFc := NewFilerConf()
|
|
require.NoError(t, concurrentFc.LoadFromBytes([]byte(readFilerConfText(client))))
|
|
require.NoError(t, concurrentFc.SetLocationConf(&filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "3d",
|
|
}))
|
|
var buf bytes.Buffer
|
|
require.NoError(t, concurrentFc.ToText(&buf))
|
|
require.NoError(t, SaveInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName, buf.Bytes()))
|
|
client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)].Attributes.Mtime = snap2.entry.Attributes.Mtime
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap2)
|
|
assert.Error(t, err, "expected the exact-content check to reject a write whose baseline content hash no longer matches")
|
|
assert.Contains(t, readFilerConfText(client), "/buckets/other/", "the concurrent writer's content must survive")
|
|
}
|
|
|
|
func TestSaveFilerConfConditionally_RejectsCreateWhenCreatedConcurrently(t *testing.T) {
|
|
client := newFakeFilerConfClient()
|
|
|
|
snap, err := readFilerConfSnapshot(context.Background(), client)
|
|
require.NoError(t, err)
|
|
require.Nil(t, snap.entry, "filer.conf should not exist yet")
|
|
|
|
// Someone else creates filer.conf after our read but before our write.
|
|
putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Collection: "b", Ttl: "3d"})
|
|
|
|
err = saveFilerConfConditionally(context.Background(), client, snap)
|
|
assert.Error(t, err, "expected a concurrent create to fail the conditional write")
|
|
assert.Contains(t, readFilerConfText(client), "/buckets/b/", "the concurrent writer's content must survive")
|
|
}
|