mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
filer.backup, filer.sync: stop sharing resume checkpoints across destinations (#10934)
* filer.backup: key the checkpoint by source path and sink destination The checkpoint id hashed only sink name + directory, so two backups to different buckets or endpoints sharing a directory layout advanced one checkpoint: whichever job was running pushed the shared offset forward, and a stopped or failing job later resumed from the other's position, silently skipping changes. Backups of different source paths to the same destination shared a checkpoint the same way. Each sink now reports a destination identity (endpoint or account, bucket or container, directory) and the checkpoint is keyed by the source path plus that identity. Reads fall back to the historical name+directory key when the new key has no value, so existing backups resume where they left off; writes go only to the new key. * filer.sync: include the target path in the offset key The offset stored on the target filer was keyed by source path and source filer signature only, so two syncs from the same source cluster and path to different directories on the same target cluster advanced one shared checkpoint, and the slower one could resume past events it never applied. The target path now participates in the key; "/" keeps the historical form, and a sync with a non-root target path falls back to the historical key once when its own key has no value yet. * join checkpoint key fields with NUL so they cannot alias A path or configuration value spelling out the separator could concatenate two different field tuples to the same checkpoint key. NUL cannot appear in a CLI path argument or any sane configuration value, making the encoding injective.
This commit is contained in:
1 parent
4a2879abad
commit
b3be2f5449
14 files changed
+362
-21
No files matched your search
+39
-12
@@ -74,6 +74,7 @@ type syncState struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
targetFiler pb.ServerAddress
|
||||
sourcePath string
|
||||
targetPath string
|
||||
sourceFilerSignature int32
|
||||
}
|
||||
|
||||
@@ -217,7 +218,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
if offsetTsNs == 0 {
|
||||
return
|
||||
}
|
||||
if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath), state.sourceFilerSignature, offsetTsNs); err != nil {
|
||||
if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath, state.targetPath), state.sourceFilerSignature, offsetTsNs); err != nil {
|
||||
glog.Errorf("failed to save checkpoint for %s on shutdown: %v", name, err)
|
||||
} else {
|
||||
glog.V(0).Infof("saved checkpoint for %s on shutdown: %v", name, time.Unix(0, offsetTsNs))
|
||||
@@ -231,7 +232,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
go func() {
|
||||
// a->b
|
||||
// set synchronization start timestamp to offset
|
||||
initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
|
||||
initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath, *syncOptions.bPath))
|
||||
if initOffsetError != nil {
|
||||
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
|
||||
os.Exit(2)
|
||||
@@ -273,7 +274,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
if !*syncOptions.isActivePassive {
|
||||
// b->a
|
||||
// set synchronization start timestamp to offset
|
||||
initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
|
||||
initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath, *syncOptions.aPath))
|
||||
if initOffsetError != nil {
|
||||
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
|
||||
os.Exit(2)
|
||||
@@ -339,7 +340,10 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
|
||||
|
||||
// if first time, start from now
|
||||
// if has previously synced, resume from that point of time
|
||||
sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
|
||||
// the historical key ignored the target path; falling back to it lets a
|
||||
// sync that predates target-scoped keys survive the upgrade
|
||||
signaturePrefix := getSignaturePrefixByPath(sourcePath, targetPath)
|
||||
sourceFilerOffsetTsNs, err := getOffsetWithFallback(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, getSignaturePrefixByPath(sourcePath, "/"), sourceFilerSignature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -387,6 +391,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
|
||||
grpcDialOption: targetGrpcDialOption,
|
||||
targetFiler: targetFiler,
|
||||
sourcePath: sourcePath,
|
||||
targetPath: targetPath,
|
||||
sourceFilerSignature: sourceFilerSignature,
|
||||
})
|
||||
}
|
||||
@@ -420,7 +425,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
|
||||
lastProgressedTsNs = offsetTsNs
|
||||
// collect synchronous offset
|
||||
statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
|
||||
return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
|
||||
return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, offsetTsNs)
|
||||
})
|
||||
|
||||
prefix := sourcePath
|
||||
@@ -452,14 +457,22 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
|
||||
|
||||
}
|
||||
|
||||
// When each business is distinguished according to path, and offsets need to be maintained separately.
|
||||
func getSignaturePrefixByPath(path string) string {
|
||||
// compatible historical version
|
||||
if path == "/" {
|
||||
return SyncKeyPrefix
|
||||
} else {
|
||||
return SyncKeyPrefix + path
|
||||
// Offsets are kept per (source path, target path): two syncs from the same
|
||||
// source to different directories on the same target filer see the same events
|
||||
// but progress independently, so a shared key would let one push the other
|
||||
// past events it never processed. "/" contributes nothing on either side,
|
||||
// keeping the historical key form so existing deployments resume unchanged.
|
||||
func getSignaturePrefixByPath(sourcePath, targetPath string) string {
|
||||
prefix := SyncKeyPrefix
|
||||
if sourcePath != "/" {
|
||||
prefix += sourcePath
|
||||
}
|
||||
if targetPath != "/" {
|
||||
// NUL cannot occur in a path, so the combined key can alias neither a
|
||||
// source-only key nor another (source, target) pair
|
||||
prefix += "\x00" + targetPath
|
||||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
|
||||
@@ -489,6 +502,20 @@ func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
|
||||
|
||||
}
|
||||
|
||||
// getOffsetWithFallback reads the offset under the current key, falling back
|
||||
// to the historical key when the current one has no value yet, so streams
|
||||
// created before a checkpoint key-scheme change resume where they left off
|
||||
// instead of replaying from zero. Writes must go only to the current key:
|
||||
// keeping the historical key warm would re-create the very sharing between
|
||||
// streams that a key-scheme change separates.
|
||||
func getOffsetWithFallback(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, legacySignaturePrefix string, legacySignature int32) (int64, error) {
|
||||
lastOffsetTsNs, err := getOffset(grpcDialOption, filer, signaturePrefix, signature)
|
||||
if err == nil && lastOffsetTsNs == 0 {
|
||||
lastOffsetTsNs, err = getOffset(grpcDialOption, filer, legacySignaturePrefix, legacySignature)
|
||||
}
|
||||
return lastOffsetTsNs, err
|
||||
}
|
||||
|
||||
func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
|
||||
return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
|
||||
Reference in new issue
Block a user