mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
Chris Lu
b3be2f5449
* filer.backup: key the checkpoint by source path and sink destination The checkpoint id hashed only sink name + directory, so two backups to different buckets or endpoints sharing a directory layout advanced one checkpoint: whichever job was running pushed the shared offset forward, and a stopped or failing job later resumed from the other's position, silently skipping changes. Backups of different source paths to the same destination shared a checkpoint the same way. Each sink now reports a destination identity (endpoint or account, bucket or container, directory) and the checkpoint is keyed by the source path plus that identity. Reads fall back to the historical name+directory key when the new key has no value, so existing backups resume where they left off; writes go only to the new key. * filer.sync: include the target path in the offset key The offset stored on the target filer was keyed by source path and source filer signature only, so two syncs from the same source cluster and path to different directories on the same target cluster advanced one shared checkpoint, and the slower one could resume past events it never applied. The target path now participates in the key; "/" keeps the historical form, and a sync with a non-root target path falls back to the historical key once when its own key has no value yet. * join checkpoint key fields with NUL so they cannot alias A path or configuration value spelling out the separator could concatenate two different field tuples to the same checkpoint key. NUL cannot appear in a CLI path argument or any sane configuration value, making the encoding injective.
150 lines
3.4 KiB
Go
150 lines
3.4 KiB
Go
package B2Sink
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/kurin/blazer/b2"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
|
|
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
|
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
type B2Sink struct {
|
|
client *b2.Client
|
|
bucket string
|
|
dir string
|
|
filerSource *source.FilerSource
|
|
isIncremental bool
|
|
}
|
|
|
|
func init() {
|
|
sink.Sinks = append(sink.Sinks, &B2Sink{})
|
|
}
|
|
|
|
func (g *B2Sink) GetName() string {
|
|
return "backblaze"
|
|
}
|
|
|
|
func (g *B2Sink) GetSinkToDirectory() string {
|
|
return g.dir
|
|
}
|
|
|
|
func (g *B2Sink) GetDestinationIdentity() string {
|
|
return g.bucket + "\x00" + g.dir
|
|
}
|
|
|
|
func (g *B2Sink) IsIncremental() bool {
|
|
return g.isIncremental
|
|
}
|
|
|
|
func (g *B2Sink) Initialize(configuration util.Configuration, prefix string) error {
|
|
g.isIncremental = configuration.GetBool(prefix + "is_incremental")
|
|
return g.initialize(
|
|
configuration.GetString(prefix+"b2_account_id"),
|
|
configuration.GetString(prefix+"b2_master_application_key"),
|
|
configuration.GetString(prefix+"bucket"),
|
|
configuration.GetString(prefix+"directory"),
|
|
)
|
|
}
|
|
|
|
func (g *B2Sink) SetSourceFiler(s *source.FilerSource) {
|
|
g.filerSource = s
|
|
}
|
|
|
|
func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
|
|
client, err := b2.NewClient(context.Background(), accountId, accountKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
g.client = client
|
|
g.bucket = bucket
|
|
g.dir = dir
|
|
|
|
return nil
|
|
}
|
|
|
|
func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
|
|
|
|
key = cleanKey(key)
|
|
|
|
if isDirectory {
|
|
key = key + "/"
|
|
}
|
|
|
|
bucket, err := g.client.Bucket(context.Background(), g.bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
targetObject := bucket.Object(key)
|
|
|
|
err = targetObject.Delete(context.Background())
|
|
if err != nil {
|
|
// b2_download_file_by_name: 404: File with such name does not exist.
|
|
if strings.Contains(err.Error(), ": 404:") {
|
|
return nil
|
|
}
|
|
}
|
|
return err
|
|
|
|
}
|
|
|
|
func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
|
|
|
|
key = cleanKey(key)
|
|
|
|
if entry.IsDirectory {
|
|
return nil
|
|
}
|
|
|
|
totalSize := filer.FileSize(entry)
|
|
chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
|
|
|
|
bucket, err := g.client.Bucket(context.Background(), g.bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
targetObject := bucket.Object(key)
|
|
writer := targetObject.NewWriter(context.Background())
|
|
defer writer.Close()
|
|
|
|
writeFunc := func(data []byte) error {
|
|
_, writeErr := writer.Write(data)
|
|
return writeErr
|
|
}
|
|
|
|
if len(entry.Content) > 0 {
|
|
content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
|
|
if err != nil {
|
|
return fmt.Errorf("decrypt inline SSE content: %w", err)
|
|
}
|
|
return writeFunc(content)
|
|
}
|
|
|
|
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
func (g *B2Sink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
|
|
key = cleanKey(key)
|
|
return true, g.CreateEntry(key, newEntry, signatures)
|
|
}
|
|
|
|
func cleanKey(key string) string {
|
|
if strings.HasPrefix(key, "/") {
|
|
key = key[1:]
|
|
}
|
|
return key
|
|
}
|