Files
seaweedfs/weed/replication/sink/filersink/filer_sink.go
T
Chris LuandClaude Opus 5.5 cadb614630 filer.sync: stop chunk retries with the same supersession check the skip uses (#12367)
* filer.sync: pass the replayed entry down to the chunk retry gates

The fetch, manifest-resolve, and manifest-upload retry gates only received the
replayed version's mtime, which is not enough to order it against the live
source entry. Pass the entry itself; the gates still compare mtime only, so
behavior is unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.sync: stop chunk retries with the same supersession check the skip uses

The fetch, manifest-resolve, and manifest-upload retry gates stopped early only
for a gone or strictly newer source entry. A transient error therefore kept
retrying a version the source had rewritten under an older mtime until the
read recovered, while a live lookup that lags the replay could stop the
retries on its newer mtime alone. The error handler that runs once a gate stops
already decides with sourceMovedOn, which orders by commit timestamp and
compares chunk sets; the gates now use it too, and hasSourceNewerVersion has no
callers left.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.sync: a lagging live lookup cannot prove a chunk dropped

A peer filer's event reaches subscribers before the source store applies it,
so the live lookup can return the version the replay replaces. That entry
never listed the replay's new chunk, so sourceDroppedChunk read a 404 on it as
an obsolete version and cut the missing-chunk wait short. Like sourceVersion,
it now ignores a live entry committed before the replayed version.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.sync: test that a replicate failure on a superseded version is skipped

Covers onReplicateChunkError with a transient error for a deleted source entry,
a later commit with an older mtime, and an unchanged source, and points the
unconfirmed-supersession test comment at it instead of the helper test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer.sync: test the manifest gate against a missing volume and a lagging lookup

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 10:43:22 -08:00

915 lines
35 KiB
Go

package filersink
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"sort"
"strings"
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
// ChunkTransferSnapshot is a lock-free, copyable view of a chunk transfer's
// progress, returned by ActiveTransfers.
type ChunkTransferSnapshot struct {
ChunkFileId string
Path string
BytesReceived int64
Status string // "downloading", "uploading", or "waiting 10s" etc.
LastErr string
}
// ChunkTransferStatus tracks the progress of a single chunk being replicated.
// Fields are guarded by mu: ChunkFileId and Path are immutable after creation,
// while BytesReceived, Status, and LastErr are updated by fetchAndWrite and
// read by ActiveTransfers.
type ChunkTransferStatus struct {
mu sync.RWMutex
ChunkTransferSnapshot
}
type FilerSink struct {
filerSource *source.FilerSource
grpcAddress string
dir string
replication string
collection string
ttlSec int32
diskType string
dataCenter string
grpcDialOption grpc.DialOption
address string
writeChunkByFiler bool
isIncremental bool
executor *util.LimitedConcurrentExecutor
signature int32
activeTransfers sync.Map // chunkFileId -> *ChunkTransferStatus
uploader *operation.Uploader
jwtForFiler security.FilerJwtProvider
// lastServedFileId is the most recent chunk the source did serve, the probe
// sourceStillServesChunks re-checks before writing an entry off.
lastServedFileId atomic.Pointer[string]
missingVolumes sync.Map // source volume id -> time.Time first found unlocatable
// missingNeedles records when each file id first came back not-found from an
// answering volume: only that needle serving again resets it, so a live
// volume's other needles neither shorten nor restart the wait.
missingNeedles sync.Map // source file id -> time.Time first answered not-found
}
func init() {
sink.Sinks = append(sink.Sinks, &FilerSink{})
}
func (fs *FilerSink) GetName() string {
return "filer"
}
func (fs *FilerSink) GetSinkToDirectory() string {
return fs.dir
}
func (fs *FilerSink) GetDestinationIdentity() string {
return fs.grpcAddress + "\x00" + fs.dir
}
func (fs *FilerSink) IsIncremental() bool {
return fs.isIncremental
}
func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error {
fs.isIncremental = configuration.GetBool(prefix + "is_incremental")
fs.dataCenter = configuration.GetString(prefix + "dataCenter")
fs.signature = util.RandomInt32()
return fs.DoInitialize(
"",
configuration.GetString(prefix+"grpcAddress"),
configuration.GetString(prefix+"directory"),
configuration.GetString(prefix+"replication"),
configuration.GetString(prefix+"collection"),
configuration.GetInt(prefix+"ttlSec"),
configuration.GetString(prefix+"disk"),
security.LoadClientTLS(util.GetViper(), "grpc.client"),
false)
}
func (fs *FilerSink) SetSourceFiler(s *source.FilerSource) {
fs.filerSource = s
}
// SetUploader sets a custom uploader for this sink, used when the target
// cluster requires different TLS certificates than the global config.
// Must be called during initialization, before any replication goroutines
// start, since it writes fs.uploader without synchronization.
func (fs *FilerSink) SetUploader(uploader *operation.Uploader) {
fs.uploader = uploader
}
// SetFilerJwtProvider sets the filer API credential for proxied chunk writes.
// Must be called during initialization, before any replication goroutines start.
func (fs *FilerSink) SetFilerJwtProvider(provider security.FilerJwtProvider) {
fs.jwtForFiler = provider
}
func (fs *FilerSink) getUploader() (*operation.Uploader, error) {
if fs.uploader != nil {
return fs.uploader, nil
}
return operation.NewUploader()
}
func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
replication string, collection string, ttlSec int, diskType string, grpcDialOption grpc.DialOption, writeChunkByFiler bool) (err error) {
fs.address = address
if fs.address == "" {
fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
}
fs.grpcAddress = grpcAddress
fs.dir = dir
fs.replication = replication
fs.collection = collection
fs.ttlSec = int32(ttlSec)
fs.diskType = diskType
fs.grpcDialOption = grpcDialOption
fs.writeChunkByFiler = writeChunkByFiler
fs.executor = util.NewLimitedConcurrentExecutor(32)
return nil
}
// SetChunkConcurrency replaces the chunk replication executor with one using the
// given concurrency limit. Must be called during initialization, before any
// replication goroutines start, since it replaces fs.executor without
// synchronization.
func (fs *FilerSink) SetChunkConcurrency(concurrency int) {
if concurrency > 0 {
fs.executor = util.NewLimitedConcurrentExecutor(concurrency)
}
}
// ActiveTransfers returns an immutable snapshot of all in-progress chunk transfers.
func (fs *FilerSink) ActiveTransfers() []ChunkTransferSnapshot {
var transfers []ChunkTransferSnapshot
fs.activeTransfers.Range(func(key, value any) bool {
t := value.(*ChunkTransferStatus)
t.mu.RLock()
transfers = append(transfers, t.ChunkTransferSnapshot)
t.mu.RUnlock()
return true
})
return transfers
}
func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
return fs.deleteEntry(key, deleteIncludeChunks, false, signatures)
}
func (fs *FilerSink) DeleteEntryKeepingRemoteObject(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
return fs.deleteEntry(key, deleteIncludeChunks, true, signatures)
}
func (fs *FilerSink) deleteEntry(key string, deleteIncludeChunks, keepRemoteObject bool, signatures []int32) error {
dir, name := util.FullPath(key).DirAndName()
glog.V(4).Infof("delete entry: %v", key)
err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := filer_pb.DoRemoveRequest(context.Background(), client, &filer_pb.DeleteEntryRequest{
Directory: dir,
Name: name,
IsDeleteData: deleteIncludeChunks,
IsRecursive: true,
IgnoreRecursiveError: true,
IsFromOtherCluster: true,
Signatures: signatures,
KeepRemoteObject: keepRemoteObject,
})
return err
})
if err != nil {
glog.V(0).Infof("delete entry %s: %v", key, err)
return fmt.Errorf("delete entry %s: %w", key, err)
}
return nil
}
var _ sink.EntryMover = (*FilerSink)(nil)
// MoveEntry relocates oldKey to newKey on the target filer via AtomicRenameEntry:
// a metadata-only move that relocates a whole subtree in one transaction, so a
// directory rename never leaves descendants missing and chunks are neither
// re-copied nor leaked.
//
// When the move fails because the old path is genuinely gone on the sink — a
// descendant the parent rename already relocated, or one never replicated —
// there is nothing to move, so it creates the new path instead (CreateEntry
// short-circuits when the entry is already there, and never deletes). Existence
// is re-checked with a direct lookup rather than inferred from the rename error,
// so a rolled-back move that left the old entry intact propagates for retry
// instead of being mistaken for "gone".
func (fs *FilerSink) MoveEntry(oldKey, newKey string, oldEntry, newEntry *filer_pb.Entry, signatures []int32) error {
oldDir, oldName := util.FullPath(oldKey).DirAndName()
newDir, newName := util.FullPath(newKey).DirAndName()
err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := client.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{
OldDirectory: oldDir,
OldName: oldName,
NewDirectory: newDir,
NewName: newName,
Signatures: signatures,
})
return err
})
if err == nil {
// The rename relocates whatever version the destination held, which can
// predate the event's snapshot — e.g. a queued update skipped after the
// source already moved to the new name. Reconcile content the same way
// an in-place update does.
if oldEntry == nil {
return nil
}
_, uerr := fs.UpdateEntry(newKey, oldEntry, newDir, newEntry, true, signatures)
return uerr
}
if missing, lookupErr := fs.entryMissing(oldKey); lookupErr == nil && missing {
glog.V(2).Infof("move %s => %s: old path gone, creating %s", oldKey, newKey, newKey)
return fs.CreateEntry(newKey, newEntry, signatures)
}
return fmt.Errorf("move %s => %s: %w", oldKey, newKey, err)
}
// entryMissing reports whether key has no entry on the target filer. A lookup
// not-found (sentinel or the gRPC string form) means missing; any other lookup
// error is returned so the caller does not treat an unknown state as missing.
func (fs *FilerSink) entryMissing(key string) (bool, error) {
dir, name := util.FullPath(key).DirAndName()
missing := false
err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lookupErr == nil {
return nil
}
// The string check is a deliberate compatibility fallback: a cross-cluster
// gRPC error often arrives as a plain status string that no longer wraps the
// filer_pb.ErrNotFound sentinel, so errors.Is alone would miss it.
if errors.Is(lookupErr, filer_pb.ErrNotFound) || strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
missing = true
return nil
}
return lookupErr
})
return missing, err
}
func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
return fs.createEntry(key, entry, signatures, true)
}
// CreateEntryLatest writes an entry the caller already resolved as the current
// source state, so the stale-replay check against the key's own source path
// does not apply.
func (fs *FilerSink) CreateEntryLatest(key string, entry *filer_pb.Entry, signatures []int32) error {
return fs.createEntry(key, entry, signatures, false)
}
func (fs *FilerSink) createEntry(key string, entry *filer_pb.Entry, signatures []int32, checkSourceVersion bool) error {
return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
dir, name := util.FullPath(key).DirAndName()
// look up existing entry
lookupRequest := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
}
// glog.V(1).Infof("lookup: %v", lookupRequest)
if resp, err := filer_pb.LookupEntry(context.Background(), client, lookupRequest); err == nil {
// A resolved-latest write carries the caller's pick of the current
// version, which can be older than the destination copy it replaces
// (the deleted latest) — only provably identical content skips it.
skip := destinationHoldsContent(resp.Entry, entry)
if checkSourceVersion {
skip = destinationSatisfiesSource(resp.Entry, entry)
}
if skip {
glog.V(3).Infof("skip overwriting %s", key)
return nil
}
glog.Warningf("re-replicating %s: destination not current (dst %d chunks, src %d chunks)",
key, len(resp.Entry.GetChunks()), len(entry.GetChunks()))
}
// A version the source already moved past is superseded by a later
// event, so fetching its chunks only feeds a write that is obsolete on
// arrival — skip before any read. When the source holds a newer live
// version, replicate it directly: the superseding event may never be
// delivered to this sink.
if checkSourceVersion {
if live, movedOn := fs.sourceVersion(key, entry); movedOn {
if live == nil {
glog.Warningf("skip stale entry %s, source superseded before replicate", key)
return nil
}
glog.Warningf("entry %s superseded before replicate; syncing live version", key)
entry = live
}
}
replicatedChunks, err := fs.replicateChunks(context.Background(), entry.GetChunks(), key, entry, "")
if err != nil {
// don't swallow: committing mismatched bytes would propagate corruption
if errors.Is(err, errChunkSizeMismatch) {
return fs.onCorruptChunk(key, entry, err)
}
return fs.onReplicateChunkError(key, entry, err)
}
// glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks)
request := &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: &filer_pb.Entry{
Name: name,
IsDirectory: entry.IsDirectory,
Attributes: entry.Attributes,
Extended: entry.Extended,
Chunks: replicatedChunks,
Content: entry.Content,
RemoteEntry: entry.RemoteEntry,
},
IsFromOtherCluster: true,
Signatures: signatures,
}
glog.V(3).Infof("create: %s", key)
if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
glog.V(0).Infof("create entry %s: %v", key, err)
return fmt.Errorf("create entry %s: %w", key, err)
}
return nil
})
}
func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
dir, name := util.FullPath(key).DirAndName()
// read existing entry
var existingEntry *filer_pb.Entry
err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
}
glog.V(4).Infof("lookup entry: %v", request)
resp, err := filer_pb.LookupEntry(context.Background(), client, request)
if err != nil {
glog.V(0).Infof("lookup %s: %v", key, err)
return err
}
existingEntry = resp.Entry
return nil
})
if err != nil {
return false, fmt.Errorf("lookup %s: %w", key, err)
}
glog.V(4).Infof("entry %s: old %q new %q existing %q", key, oldEntry.GetName(), newEntry.GetName(), existingEntry.GetName())
// the supersession checks below must look up where the incoming entry lives
// now, which for a rename is newParentPath/newEntry.Name, not the old key.
targetKey := updatedEntryKey(key, newParentPath, newEntry)
action := chooseUpdateAction(existingEntry, oldEntry, newEntry)
if action != updateSkip {
if live, movedOn := fs.sourceVersion(targetKey, newEntry); movedOn {
if live == nil {
glog.Warningf("skip stale entry %s, source superseded before replicate", targetKey)
return true, nil
}
glog.Warningf("entry %s superseded before replicate; syncing live version", targetKey)
newEntry = live
// The live snapshot is the source's current state, not a replay:
// a newer destination mtime does not make it stale, and the
// replayed oldEntry predates it, so an incremental diff could
// resolve manifests the overwrite already vacuumed. Repair from
// the snapshot unless the destination already holds it.
action = updateRepair
if destinationHoldsContent(existingEntry, live) {
action = updateSkip
}
}
}
switch action {
case updateSkip:
// a newer, complete version already landed (out-of-order); leave it
glog.V(2).Infof("late updates %s", key)
return true, nil
case updateRepair:
glog.Warningf("repair %s: destination diverged from source (dst %d chunks, src %d chunks); re-replicating full source content",
key, len(existingEntry.GetChunks()), len(newEntry.GetChunks()))
replicatedChunks, err := fs.replicateChunks(context.Background(), newEntry.GetChunks(), targetKey, newEntry, "")
if err != nil {
if errors.Is(err, errChunkSizeMismatch) {
return true, fs.onCorruptChunk(targetKey, newEntry, err)
}
return true, fs.onReplicateChunkError(targetKey, newEntry, err)
}
existingEntry.Chunks = replicatedChunks
existingEntry.Attributes = newEntry.Attributes
existingEntry.Extended = newEntry.Extended
existingEntry.HardLinkId = newEntry.HardLinkId
existingEntry.HardLinkCounter = newEntry.HardLinkCounter
existingEntry.Content = newEntry.Content
existingEntry.RemoteEntry = newEntry.RemoteEntry
default:
// source-side chunks resolve via source filer; sink volume IDs may collide.
deletedChunks, newChunks, err := compareChunks(context.Background(), filer.LookupFn(fs.filerSource), fs.filerSource.FilerJwt(), oldEntry, newEntry)
if err != nil {
return true, fmt.Errorf("replicate %s compare chunks error: %w", key, err)
}
// delete the chunks that are deleted from the source
if deleteIncludeChunks {
// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.GetChunks(), deletedChunks)
}
// replicate the chunks that are new in the source
replicatedChunks, err := fs.replicateChunks(context.Background(), newChunks, targetKey, newEntry, "")
if err != nil {
if errors.Is(err, errChunkSizeMismatch) {
return true, fs.onCorruptChunk(targetKey, newEntry, err)
}
return true, fs.onReplicateChunkError(targetKey, newEntry, err)
}
existingEntry.Chunks = append(existingEntry.GetChunks(), replicatedChunks...)
existingEntry.Attributes = newEntry.Attributes
existingEntry.Extended = newEntry.Extended
existingEntry.HardLinkId = newEntry.HardLinkId
existingEntry.HardLinkCounter = newEntry.HardLinkCounter
existingEntry.Content = newEntry.Content
existingEntry.RemoteEntry = newEntry.RemoteEntry
}
// save updated meta data
return true, fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: newParentPath,
Entry: existingEntry,
IsFromOtherCluster: true,
Signatures: signatures,
}
if _, err := client.UpdateEntry(context.Background(), request); err != nil {
return fmt.Errorf("update existingEntry %s: %w", key, err)
}
return nil
})
}
func compareChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, filerJwtFn security.FilerJwtProvider, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) {
aData, aMeta, aErr := filer.ResolveChunkManifestWithFilerJwt(ctx, lookupFileIdFn, filerJwtFn, oldEntry.GetChunks(), 0, math.MaxInt64, nil)
if aErr != nil {
return nil, nil, aErr
}
bData, bMeta, bErr := filer.ResolveChunkManifestWithFilerJwt(ctx, lookupFileIdFn, filerJwtFn, newEntry.GetChunks(), 0, math.MaxInt64, nil)
if bErr != nil {
return nil, nil, bErr
}
deletedChunks = append(deletedChunks, filer.DoMinusChunks(aData, bData)...)
deletedChunks = append(deletedChunks, filer.DoMinusChunks(aMeta, bMeta)...)
newChunks = append(newChunks, filer.DoMinusChunks(bData, aData)...)
newChunks = append(newChunks, filer.DoMinusChunks(bMeta, aMeta)...)
return
}
// getEntryMtimeNs returns the mtime at full nanosecond precision so versions
// rewritten within the same second can be ordered. int64 ns is safe until ~2262.
func getEntryMtimeNs(entry *filer_pb.Entry) int64 {
if entry == nil || entry.Attributes == nil {
return 0
}
return entry.Attributes.Mtime*int64(1e9) + int64(entry.Attributes.MtimeNs)
}
// EntryMtimeNs exposes getEntryMtimeNs for filer.backup's supersession check.
func EntryMtimeNs(entry *filer_pb.Entry) int64 {
return getEntryMtimeNs(entry)
}
// onCorruptChunk handles a permanent chunk size-mismatch (fetched source bytes
// disagree with source metadata). If the source already holds a newer version the
// event is a stale replay whose chunk was overwritten and GC'd — skip it (lossless:
// meta events are full snapshots, so a later event re-carries the current chunks).
// Otherwise the live version is corrupt, so return the error and halt loudly.
func (fs *FilerSink) onCorruptChunk(key string, entry *filer_pb.Entry, err error) error {
if fs.sourceMovedOn(key, entry) {
glog.Warningf("skip stale entry %s with superseded corrupt chunk: %v", key, err)
return nil
}
glog.Errorf("refuse to replicate entry with corrupt chunk %s: %v", key, err)
return err
}
// onReplicateChunkError handles a non-size-mismatch replicateChunks failure.
// Skip (return nil) when the live source has moved past this version — deleted
// or strictly-newer mtime — so a later event carries the current content and the
// skip is lossless, or when the source has lost the chunk for good: it can never
// serve those bytes to anyone, and holding the offset for it only stops every
// later event from ever being checkpointed. Otherwise propagate so the offset
// stays put and the event is retried; returning nil would silently drop the file.
func (fs *FilerSink) onReplicateChunkError(key string, entry *filer_pb.Entry, err error) error {
if fs.sourceMovedOn(key, entry) {
glog.Warningf("skip stale entry %s, source superseded during replicate: %v", key, err)
return nil
}
// A volume server's own 404 names this needle, so a direct-read 404 needs no
// liveness probe; a filer-proxied 404 also covers "no volume locations" and
// could be an outage, so it still must pass the probe.
direct404 := errors.Is(err, util_http.ErrNotFound) && fs.filerSource != nil && !fs.filerSource.ProxyByFiler()
if errors.Is(err, errSourceChunkMissing) && (direct404 || fs.sourceStillServesChunks()) {
glog.Errorf("skip %s: the source cluster cannot produce its data, so it stays unreplicated: %v", key, err)
return nil
}
return fmt.Errorf("replicate entry chunks %s: %w", key, err)
}
// sourceVersion returns the source's live entry for targetPath and whether it
// moved past the replayed version: gone (live == nil), strictly newer, or
// rewritten so it no longer references the same chunks — the last is invisible
// to the mtime check, and holds whatever mtime the rewrite was stamped with,
// since timestamp-preserving writers make a newer version read older. A failed
// lookup proves nothing, and neither does a live entry committed before the
// replayed version: the lookup lags the event, as when a peer filer's event
// reaches subscribers before the local store applies it.
func (fs *FilerSink) sourceVersion(targetPath string, entry *filer_pb.Entry) (live *filer_pb.Entry, movedOn bool) {
sourcePath, ok := fs.targetPathToSourcePath(targetPath)
if !ok {
return nil, false
}
live, lookupErr := fs.sourceEntry(sourcePath)
if lookupErr == nil && live != nil && committedBefore(live, entry) {
return nil, false
}
if sourceSupersedes(sourcePath, live, lookupErr, getEntryMtimeNs(entry)) {
return live, true
}
if lookupErr != nil || live == nil {
return nil, false
}
liveIds := chunkFileIds(live.GetChunks())
replayedIds := chunkFileIds(entry.GetChunks())
if len(liveIds) != len(replayedIds) {
return live, true
}
for id := range replayedIds {
if !liveIds[id] {
return live, true
}
}
return nil, false
}
// committedBefore reports whether the source committed live before replayed.
// Both stamps come from the source's meta log, so they order versions that
// mtime cannot; an unstamped live entry predates a stamped replay.
func committedBefore(live, replayed *filer_pb.Entry) bool {
return replayed.GetCommitTsNs() > 0 && live.GetCommitTsNs() < replayed.GetCommitTsNs()
}
// sourceMovedOn reports whether the live source entry moved past the replayed
// version: gone, strictly newer, or rewritten in place so it no longer
// references the same chunks — the last is invisible to the mtime check.
func (fs *FilerSink) sourceMovedOn(targetPath string, entry *filer_pb.Entry) bool {
_, movedOn := fs.sourceVersion(targetPath, entry)
return movedOn
}
// updateAction decides how UpdateEntry reconciles an incoming source version with
// the destination's current entry.
type updateAction int
const (
// updateNormal applies the source diff: destination at or behind the source.
updateNormal updateAction = iota
// updateSkip leaves the destination untouched: a newer complete version already landed.
updateSkip
// updateRepair re-replicates full source over a destination strictly shorter than
// the source despite a newer mtime — a truncated copy from a failed replication.
updateRepair
)
// updatedEntryKey returns the sink path the incoming entry lands at: for a rename
// newParentPath/newEntry.Name, else key's parent + name. Supersession must use this,
// not the pre-rename key, which would look deleted on the source and skip a live event.
func updatedEntryKey(key, newParentPath string, newEntry *filer_pb.Entry) string {
targetDir := newParentPath
if targetDir == "" {
targetDir, _ = util.FullPath(key).DirAndName()
}
return string(util.NewFullPath(targetDir, newEntry.Name))
}
func chooseUpdateAction(existing, oldEntry, incoming *filer_pb.Entry) updateAction {
exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
if exNs > inNs {
// Destination is strictly newer: incoming is a stale/out-of-order replay of
// an older source version, so never roll the destination back to it — even
// when the newer destination is smaller (a legitimate truncate). A newer copy
// that is itself incomplete is reconciled by -initialSnapshot, not by an
// older event.
return updateSkip
}
// Destination at or behind incoming. updateNormal applies the incremental
// oldEntry->incoming diff, valid only when the destination already holds
// oldEntry's chunks. A cheap, allocation-free positional identity match proves
// that for the common (in-sync) path; only when it fails — divergence, chunk
// reordering, or an out-of-band destination — do we run the precise O(n log n)
// checks.
if existing != nil && oldEntry != nil && !destinationMatchesReference(existing, oldEntry) {
// Truncated/gapped: the destination does not cover every byte range oldEntry
// holds. Range containment (not a scalar byte count, attr FileSize, or ETag)
// catches it for any destination, and is safe at equal mtime since a complete
// destination always covers oldEntry.
if !coversReference(existing, oldEntry) {
return updateRepair
}
// Stale superseded chunk at full coverage: a chunk this backup wrote whose
// source oldEntry no longer references. Restricted to backup-written chunks
// (out-of-band rsync/direct chunks are ignored) AND to a strictly-older
// destination — at equal mtime same-second versions cannot be ordered, so
// defer rather than risk a rollback (reliable ordering is a separate fix).
if exNs < inNs && hasStaleBackupChunk(existing, oldEntry) {
return updateRepair
}
}
return updateNormal
}
// destinationHoldsContent reports whether the destination already holds exactly
// the resolved entry for a CreateEntryLatest write, where a newer destination
// mtime proves nothing: the resolved version may be older than the copy it
// replaces. Skip only when the writable metadata already matches and the content
// is identical — chunk provenance, or an equal ETag over full byte coverage.
func destinationHoldsContent(existing, reference *filer_pb.Entry) bool {
if existing == nil {
return false
}
// A skip keeps the destination's stored metadata, so identical bytes are not
// enough: the resolved version's writable attributes and extended entries
// must already match what the write would install, or a stale tag/version id
// survives under equal content.
if !sameWritableMetadata(existing, reference) {
return false
}
if destinationMatchesReference(existing, reference) {
return true
}
return sameInlineContent(existing, reference) &&
coversReference(existing, reference) &&
filer.ETag(existing) == filer.ETag(reference)
}
// sameWritableMetadata reports whether the destination already stores the metadata
// the resolved write would install. Inode and creation time are filer-assigned or
// preserved across rewrites — comparing them would re-replicate forever on fields
// a write can never change — so only the writable fields decide.
func sameWritableMetadata(existing, reference *filer_pb.Entry) bool {
exAttr, refAttr := normalizedAttributes(existing.GetAttributes()), normalizedAttributes(reference.GetAttributes())
if exAttr != nil && refAttr.GetTtlSec() == 0 {
// a TTL-less write inherits the destination's storage-rule TTL, so the
// stored value is one a rewrite reproduces, not one it would change
exAttr.TtlSec = 0
}
if !proto.Equal(exAttr, refAttr) {
return false
}
ex, ref := existing.GetExtended(), reference.GetExtended()
if len(ex) != len(ref) {
return false
}
for k, v := range ex {
refValue, found := ref[k]
if !found || !bytes.Equal(v, refValue) {
return false
}
}
return true
}
// normalizedAttributes masks the fields the destination filer assigns or preserves
// on write and applies the store's zero-value defaults (access/change time fall
// back to mtime), so the comparison covers only fields a rewrite controls.
func normalizedAttributes(attr *filer_pb.FuseAttributes) *filer_pb.FuseAttributes {
if attr == nil {
return nil
}
a := proto.Clone(attr).(*filer_pb.FuseAttributes)
a.Inode = 0
a.Crtime = 0
a.CrtimeNs = 0
if a.Atime == 0 && a.AtimeNs == 0 {
a.Atime, a.AtimeNs = a.Mtime, a.MtimeNs
}
if a.Ctime == 0 {
a.Ctime, a.CtimeNs = a.Mtime, a.MtimeNs
}
return a
}
// destinationSatisfiesSource reports whether an existing destination entry already
// satisfies the incoming source entry, so CreateEntry can skip the write. Coverage
// is checked first: a chunk-truncated destination (covering fewer bytes than the
// source) must be re-replicated even when its attr.Md5-backed ETag still matches
// the source, so ETag equality must never bypass the coverage check. A destination
// that is at least as complete is skippable when it has identical content (ETag) or
// is a newer-or-equal version.
func destinationSatisfiesSource(existing, incoming *filer_pb.Entry) bool {
if existing == nil {
return false
}
exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
if exNs > inNs {
return true // destination strictly newer: incoming is older → keep destination
}
if !destinationMatchesReference(existing, incoming) {
if !sameInlineContent(existing, incoming) {
return false // equal chunk lists hide nothing: the inline bytes differ
}
if !coversReference(existing, incoming) {
return false // missing a byte range the source covers (truncated/gapped)
}
if exNs < inNs && hasStaleBackupChunk(existing, incoming) {
return false // a provably-older destination holds a superseded backup chunk
}
}
if filer.ETag(existing) == filer.ETag(incoming) {
return true // identical content
}
return exNs >= inNs // equal mtime → keep (deferred); strictly older → re-replicate
}
// destinationMatchesReference is a cheap, allocation-free sufficient check that the
// destination already holds exactly the reference's chunks: equal count and, in
// order, each destination chunk was replicated from the corresponding reference
// chunk (SourceFileId == reference FileId) at the same byte range (Offset/Size).
// True ⇒ full coverage and no stale chunk, so updateNormal/skip is safe and the
// O(n log n) range and identity checks can be skipped — the common in-sync path.
// False is never a false positive: a reordered, diverged, or out-of-band
// (rsync/direct) destination just falls back to the precise checks.
func destinationMatchesReference(existing, reference *filer_pb.Entry) bool {
ex, ref := existing.GetChunks(), reference.GetChunks()
if len(ex) != len(ref) {
return false
}
for i := range ref {
if ex[i].SourceFileId == "" || ex[i].SourceFileId != ref[i].GetFileIdString() {
return false
}
// Require the same byte range too. replicateOneChunk copies the source
// chunk's Offset/Size verbatim, so SourceFileId identity already implies an
// identical range today; verifying it here keeps this fast path sound even
// if replication ever re-chunks (split/coalesce), instead of silently
// depending on that invariant from another file.
if ex[i].Offset != ref[i].Offset || ex[i].Size != ref[i].Size {
return false
}
}
// Identical chunk lists say nothing about content stored outside them:
// a chunk-less entry keeps its bytes in Content, a remote-only entry in the
// object RemoteEntry points at. Both must compare equal to claim a match.
return sameInlineContent(existing, reference)
}
// sameInlineContent reports whether two entries carry the same out-of-chunk
// content — inline bytes, or the same remote object — so a destination can be
// trusted to hold the reference's bytes when the chunk comparison is done.
func sameInlineContent(a, b *filer_pb.Entry) bool {
ra, rb := a.GetRemoteEntry(), b.GetRemoteEntry()
if (ra == nil) != (rb == nil) {
return false
}
if ra != nil && (ra.StorageName != rb.StorageName ||
ra.RemoteETag != rb.RemoteETag ||
ra.RemoteMtime != rb.RemoteMtime ||
ra.RemoteSize != rb.RemoteSize) {
return false
}
return bytes.Equal(a.GetContent(), b.GetContent())
}
// hasStaleBackupChunk reports whether existing holds a chunk that this backup
// wrote (SourceFileId set) whose source chunk `reference` no longer references.
// That means an overwrite/deletion event was missed: the destination still carries
// the superseded source chunk's bytes even though total coverage looks full, so the
// incremental diff cannot fix it. Chunks without SourceFileId (seeded out-of-band,
// e.g. rsync or a direct write) are ignored, so such destinations keep their normal
// path and are never re-copied on this signal.
func hasStaleBackupChunk(existing, reference *filer_pb.Entry) bool {
if existing == nil || reference == nil {
return false
}
refFids := make(map[string]bool, len(reference.GetChunks()))
for _, c := range reference.GetChunks() {
refFids[c.GetFileIdString()] = true
}
for _, c := range existing.GetChunks() {
if c.SourceFileId == "" {
continue // not backup-written → do not judge staleness
}
if !refFids[c.SourceFileId] {
return true
}
}
return false
}
// mergedIntervals returns the entry's chunk byte ranges, sorted and merged
// (overlapping or touching ranges combined). Chunk manifests are counted by their
// own offset/size span, which already covers the range they wrap, so no manifest
// resolution is needed.
func mergedIntervals(entry *filer_pb.Entry) [][2]int64 {
if entry == nil {
return nil
}
chunks := entry.GetChunks()
if len(chunks) == 0 {
return nil
}
ivs := make([][2]int64, 0, len(chunks))
for _, c := range chunks {
ivs = append(ivs, [2]int64{int64(c.Offset), int64(c.Offset) + int64(c.Size)})
}
sort.Slice(ivs, func(i, j int) bool { return ivs[i][0] < ivs[j][0] })
merged := make([][2]int64, 0, len(ivs))
for _, iv := range ivs {
if n := len(merged); n > 0 && iv[0] <= merged[n-1][1] {
if iv[1] > merged[n-1][1] {
merged[n-1][1] = iv[1]
}
continue
}
merged = append(merged, iv)
}
return merged
}
// coversReference reports whether existing's chunks cover every byte range that
// reference's chunks cover. Equal total coverage is not sufficient proof — extra
// chunks at other offsets can hide a missing range — so containment is verified
// range by range against the merged interval sets.
func coversReference(existing, reference *filer_pb.Entry) bool {
ex := mergedIntervals(existing)
i := 0
for _, r := range mergedIntervals(reference) {
lo, hi := r[0], r[1]
for lo < hi {
for i < len(ex) && ex[i][1] <= lo {
i++
}
if i >= len(ex) || ex[i][0] > lo {
return false // a byte in reference is not covered by existing
}
lo = ex[i][1]
}
}
return true
}