seaweedfs/weed/replication/replicator_test.go

package replication

import (
	"context"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
	"github.com/seaweedfs/seaweedfs/weed/replication/source"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

var _ sink.ReplicationSink = (*recordingSink)(nil)

type deleteCall struct {
	key         string
	isDirectory bool
}

type createCall struct {
	key string
}

type updateCall struct {
	key           string
	newParentPath string
}

type recordingSink struct {
	name                string
	sinkToDirectory     string
	incremental         bool
	updateFoundExisting bool

	deleteCalls []deleteCall
	createCalls []createCall
	updateCalls []updateCall

	// ordered records the sink method names in call order so tests can assert
	// create-before-delete sequencing.
	ordered []string
}

func (s *recordingSink) GetName() string {
	return s.name
}

func (s *recordingSink) Initialize(util.Configuration, string) error {
	return nil
}

func (s *recordingSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
	s.deleteCalls = append(s.deleteCalls, deleteCall{key: key, isDirectory: isDirectory})
	s.ordered = append(s.ordered, "delete")
	return nil
}

func (s *recordingSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
	s.createCalls = append(s.createCalls, createCall{key: key})
	s.ordered = append(s.ordered, "create")
	return nil
}

func (s *recordingSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
	s.updateCalls = append(s.updateCalls, updateCall{key: key, newParentPath: newParentPath})
	s.ordered = append(s.ordered, "update")
	return s.updateFoundExisting, nil
}

func (s *recordingSink) GetSinkToDirectory() string {
	return s.sinkToDirectory
}

func (s *recordingSink) GetDestinationIdentity() string {
	return s.sinkToDirectory
}

func (s *recordingSink) SetSourceFiler(*source.FilerSource) {}

func (s *recordingSink) IsIncremental() bool {
	return s.incremental
}

type moveCall struct {
	oldKey string
	newKey string
}

// movingSink is a recordingSink that also implements sink.EntryMover, modeling
// a sink (like the filer) with a native atomic move.
type movingSink struct {
	*recordingSink
	moveCalls []moveCall
}

var _ sink.EntryMover = (*movingSink)(nil)

func (s *movingSink) MoveEntry(oldKey, newKey string, oldEntry, newEntry *filer_pb.Entry, signatures []int32) error {
	s.moveCalls = append(s.moveCalls, moveCall{oldKey: oldKey, newKey: newKey})
	s.ordered = append(s.ordered, "move")
	return nil
}

func TestReplicateRenameUsesTargetKeyForNonFilerSink(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "renamed.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.updateCalls) != 0 {
		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want create before delete %v", got, want)
	}
}

// A real move to a filer sink also goes through create-then-delete (the
// filer-sink special-case that previously routed renames to UpdateEntry is
// gone): UpdateEntry cannot move an entry across paths.
// A sink without a native move falls back to create-then-delete for a rename.
func TestReplicateRenameWithoutMoverUsesCreateThenDelete(t *testing.T) {
	s := &recordingSink{
		name:                "s3",
		sinkToDirectory:     "/dest",
		updateFoundExisting: true,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "renamed.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.updateCalls) != 0 {
		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want create before delete %v", got, want)
	}
}

// A sink with a native move (the filer) relocates a rename via MoveEntry — never
// create-then-delete, so a failed copy can't delete the only valid destination.
func TestReplicateRenameUsesMoveEntryWhenSupported(t *testing.T) {
	s := &movingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry:      &filer_pb.Entry{Name: "file.txt"},
		NewEntry:      &filer_pb.Entry{Name: "renamed.txt"},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.moveCalls) != 1 || s.moveCalls[0].oldKey != "/dest/old/file.txt" || s.moveCalls[0].newKey != "/dest/new/renamed.txt" {
		t.Fatalf("move calls = %+v, want one move /dest/old/file.txt => /dest/new/renamed.txt", s.moveCalls)
	}
	if len(s.createCalls) != 0 || len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("native move must not create/delete/update: creates=%v deletes=%v updates=%v", s.createCalls, s.deleteCalls, s.updateCalls)
	}
}

// An in-place update (same parent + same name, so oldSinkKey == newSinkKey)
// still routes to UpdateEntry rather than create-then-delete.
func TestReplicateInPlaceUpdateUsesUpdateEntry(t *testing.T) {
	s := &recordingSink{
		name:                "filer",
		sinkToDirectory:     "/dest",
		updateFoundExisting: true,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 456,
			},
		},
		NewParentPath: "/source/dir",
	})
	if err != nil {
		t.Fatalf("Replicate in-place update: %v", err)
	}

	if len(s.updateCalls) != 1 {
		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
	}
	if s.updateCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("update key = %q, want /dest/dir/file.txt", s.updateCalls[0].key)
	}
	if s.updateCalls[0].newParentPath != "/dest/dir" {
		t.Fatalf("update newParentPath = %q, want /dest/dir", s.updateCalls[0].newParentPath)
	}
	if len(s.deleteCalls) != 0 || len(s.createCalls) != 0 {
		t.Fatalf("unexpected delete/create calls: deletes=%+v creates=%+v", s.deleteCalls, s.createCalls)
	}
}

// When the in-place update finds no existing entry to update, fall back to
// delete-then-create at the same key.
func TestReplicateInPlaceUpdateFallbackCreates(t *testing.T) {
	s := &recordingSink{
		name:                "filer",
		sinkToDirectory:     "/dest",
		updateFoundExisting: false,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 456,
			},
		},
		NewParentPath: "/source/dir",
	})
	if err != nil {
		t.Fatalf("Replicate in-place update fallback: %v", err)
	}

	if len(s.updateCalls) != 1 {
		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("delete calls = %+v, want same sink key", s.deleteCalls)
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("create calls = %+v, want same sink key", s.createCalls)
	}
	if got, want := s.ordered, []string{"update", "delete", "create"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want %v", got, want)
	}
}

func equalStrings(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
	tests := []struct {
		name      string
		candidate string
		other     string
		expected  bool
	}{
		{name: "equal", candidate: "/foo", other: "/foo", expected: true},
		{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
		{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
		{name: "root", candidate: "/foo/bar", other: "/", expected: true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
				t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
			}
		})
	}
}

func TestReplicateRenameOutToSiblingPrefixBecomesDelete(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/foo"},
	}

	err := r.Replicate(context.Background(), "/foo/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/foobar/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename out to sibling prefix: %v", err)
	}

	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if len(s.createCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("unexpected create/update calls: creates=%+v updates=%+v", s.createCalls, s.updateCalls)
	}
}

func TestReplicateRenameFromExcludedDirBecomesCreate(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:        s,
		source:      &source.FilerSource{Dir: "/foo"},
		excludeDirs: []string{"/foo/excluded"},
	}

	err := r.Replicate(context.Background(), "/foo/excluded/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/foo/live",
	})
	if err != nil {
		t.Fatalf("Replicate rename from excluded dir: %v", err)
	}

	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/live/file.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("unexpected delete/update calls: deletes=%+v updates=%+v", s.deleteCalls, s.updateCalls)
	}
}

type keepingSink struct {
	*recordingSink
	keepCalls []deleteCall
}

var _ sink.MetadataOnlyDeleter = (*keepingSink)(nil)

func (s *keepingSink) DeleteEntryKeepingRemoteObject(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
	s.keepCalls = append(s.keepCalls, deleteCall{key: key, isDirectory: isDirectory})
	return nil
}

func TestReplicateMetadataOnlyDeleteKeepsTheRemoteObject(t *testing.T) {
	s := &keepingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
	r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
	deleteOf := func(name string, extended map[string][]byte) *filer_pb.EventNotification {
		return &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}, Extended: extended}}
	}

	if err := r.Replicate(context.Background(), "/source/dir/kept.bin", deleteOf("kept.bin", map[string][]byte{filer.ExtKeepRemoteObjectKey: []byte("true")})); err != nil {
		t.Fatal(err)
	}
	if err := r.Replicate(context.Background(), "/source/dir/gone.bin", deleteOf("gone.bin", nil)); err != nil {
		t.Fatal(err)
	}

	if len(s.keepCalls) != 1 || s.keepCalls[0].key != "/dest/dir/kept.bin" {
		t.Fatalf("metadata-only deletes = %+v, want /dest/dir/kept.bin", s.keepCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/gone.bin" {
		t.Fatalf("ordinary deletes = %+v, want /dest/dir/gone.bin", s.deleteCalls)
	}
}

type versionedSink struct {
	*recordingSink
	versionCalls []*filer_pb.Entry
}

var _ sink.VersionedDeleter = (*versionedSink)(nil)

func (s *versionedSink) DeleteEntryVersion(key string, oldEntry *filer_pb.Entry, tsNs int64, deleteIncludeChunks bool, signatures []int32) error {
	s.versionCalls = append(s.versionCalls, oldEntry)
	return s.DeleteEntry(key, oldEntry.IsDirectory, deleteIncludeChunks, signatures)
}

func TestReplicateDeletesHandTheDeletedVersionToTheSink(t *testing.T) {
	s := &versionedSink{recordingSink: &recordingSink{name: "local", sinkToDirectory: "/dest"}}
	r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
	deleted := &filer_pb.Entry{Name: "gone.bin", CommitTsNs: 100, Attributes: &filer_pb.FuseAttributes{}}
	if err := r.Replicate(context.Background(), "/source/dir/gone.bin", &filer_pb.EventNotification{OldEntry: deleted}); err != nil {
		t.Fatal(err)
	}
	renamed := &filer_pb.Entry{Name: "old.bin", CommitTsNs: 200, Attributes: &filer_pb.FuseAttributes{}}
	if err := r.Replicate(context.Background(), "/source/dir/old.bin", &filer_pb.EventNotification{
		OldEntry:      renamed,
		NewEntry:      &filer_pb.Entry{Name: "new.bin", Attributes: &filer_pb.FuseAttributes{}},
		NewParentPath: "/source/dir",
	}); err != nil {
		t.Fatal(err)
	}

	if len(s.versionCalls) != 2 || s.versionCalls[0] != deleted || s.versionCalls[1] != renamed {
		t.Fatalf("versioned deletes = %v, want the deleted and the renamed entries", s.versionCalls)
	}
	if len(s.deleteCalls) != 2 || s.deleteCalls[0].key != "/dest/dir/gone.bin" || s.deleteCalls[1].key != "/dest/dir/old.bin" {
		t.Fatalf("deletes = %+v", s.deleteCalls)
	}
}