filer.backup, filer.sync: stop sharing resume checkpoints across destinations (#10934)

* filer.backup: key the checkpoint by source path and sink destination

The checkpoint id hashed only sink name + directory, so two backups to
different buckets or endpoints sharing a directory layout advanced one
checkpoint: whichever job was running pushed the shared offset forward,
and a stopped or failing job later resumed from the other's position,
silently skipping changes. Backups of different source paths to the same
destination shared a checkpoint the same way.

Each sink now reports a destination identity (endpoint or account,
bucket or container, directory) and the checkpoint is keyed by the
source path plus that identity. Reads fall back to the historical
name+directory key when the new key has no value, so existing backups
resume where they left off; writes go only to the new key.

* filer.sync: include the target path in the offset key

The offset stored on the target filer was keyed by source path and
source filer signature only, so two syncs from the same source cluster
and path to different directories on the same target cluster advanced
one shared checkpoint, and the slower one could resume past events it
never applied. The target path now participates in the key; "/" keeps
the historical form, and a sync with a non-root target path falls back
to the historical key once when its own key has no value yet.

* join checkpoint key fields with NUL so they cannot alias

A path or configuration value spelling out the separator could
concatenate two different field tuples to the same checkpoint key.
NUL cannot appear in a CLI path argument or any sane configuration
value, making the encoding injective.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-08-24 19:30:20 -07:00
1 parent 4a2879abad
commit b3be2f5449
14 files changed
+362 -21

No files matched your search

+39 -12
View File
@@ -74,6 +74,7 @@ type syncState struct {
grpcDialOption grpc.DialOption
targetFiler pb.ServerAddress
sourcePath string
targetPath string
sourceFilerSignature int32
}
@@ -217,7 +218,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
if offsetTsNs == 0 {
return
}
if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath), state.sourceFilerSignature, offsetTsNs); err != nil {
if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath, state.targetPath), state.sourceFilerSignature, offsetTsNs); err != nil {
glog.Errorf("failed to save checkpoint for %s on shutdown: %v", name, err)
} else {
glog.V(0).Infof("saved checkpoint for %s on shutdown: %v", name, time.Unix(0, offsetTsNs))
@@ -231,7 +232,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
go func() {
// a->b
// set synchronization start timestamp to offset
initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath, *syncOptions.bPath))
if initOffsetError != nil {
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
os.Exit(2)
@@ -273,7 +274,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
if !*syncOptions.isActivePassive {
// b->a
// set synchronization start timestamp to offset
initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath, *syncOptions.aPath))
if initOffsetError != nil {
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
os.Exit(2)
@@ -339,7 +340,10 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
// if first time, start from now
// if has previously synced, resume from that point of time
sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
// the historical key ignored the target path; falling back to it lets a
// sync that predates target-scoped keys survive the upgrade
signaturePrefix := getSignaturePrefixByPath(sourcePath, targetPath)
sourceFilerOffsetTsNs, err := getOffsetWithFallback(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, getSignaturePrefixByPath(sourcePath, "/"), sourceFilerSignature)
if err != nil {
return err
}
@@ -387,6 +391,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
grpcDialOption: targetGrpcDialOption,
targetFiler: targetFiler,
sourcePath: sourcePath,
targetPath: targetPath,
sourceFilerSignature: sourceFilerSignature,
})
}
@@ -420,7 +425,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
lastProgressedTsNs = offsetTsNs
// collect synchronous offset
statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, offsetTsNs)
})
prefix := sourcePath
@@ -452,14 +457,22 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
}
// When each business is distinguished according to path, and offsets need to be maintained separately.
func getSignaturePrefixByPath(path string) string {
// compatible historical version
if path == "/" {
return SyncKeyPrefix
} else {
return SyncKeyPrefix + path
// Offsets are kept per (source path, target path): two syncs from the same
// source to different directories on the same target filer see the same events
// but progress independently, so a shared key would let one push the other
// past events it never processed. "/" contributes nothing on either side,
// keeping the historical key form so existing deployments resume unchanged.
func getSignaturePrefixByPath(sourcePath, targetPath string) string {
prefix := SyncKeyPrefix
if sourcePath != "/" {
prefix += sourcePath
}
if targetPath != "/" {
// NUL cannot occur in a path, so the combined key can alias neither a
// source-only key nor another (source, target) pair
prefix += "\x00" + targetPath
}
return prefix
}
func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
@@ -489,6 +502,20 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
}
// getOffsetWithFallback reads the offset under the current key, falling back
// to the historical key when the current one has no value yet, so streams
// created before a checkpoint key-scheme change resume where they left off
// instead of replaying from zero. Writes must go only to the current key:
// keeping the historical key warm would re-create the very sharing between
// streams that a key-scheme change separates.
func getOffsetWithFallback(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, legacySignaturePrefix string, legacySignature int32) (int64, error) {
lastOffsetTsNs, err := getOffset(grpcDialOption, filer, signaturePrefix, signature)
if err == nil && lastOffsetTsNs == 0 {
lastOffsetTsNs, err = getOffset(grpcDialOption, filer, legacySignaturePrefix, legacySignature)
}
return lastOffsetTsNs, err
}
func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {