mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 03:43:41 +01:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
25896cf051 | ||
|
|
8cf8abf9f6 |
No files matched your search
@@ -2,11 +2,13 @@ package command
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -936,18 +938,22 @@ func readRemoteStorageConf(filerClient filer_pb.FilerClient, storageName string)
|
||||
}
|
||||
|
||||
// isSuperseded reports whether the filer has moved past the entry an event
|
||||
// described: it is deleted, or it no longer references every chunk the event
|
||||
// named. Those are the chunks the filer deletes when an entry is updated, so
|
||||
// they are gone from the volume servers, or about to be, and no retry of the
|
||||
// upload can succeed. Chunks are compared by file id, not content: a rewrite
|
||||
// stores even identical bytes under new ids and drops the old ones. A replay
|
||||
// from an earlier offset (-timeAgo) re-emits such events. Failing one holds the
|
||||
// sync offset before it, so every restart of the subscription replays it into
|
||||
// the same dead chunks, and progress on everything after it in the log is never
|
||||
// persisted. Skipping is safe: the event that superseded this one follows in
|
||||
// the log, and the delete removes the remote object or the rewrite uploads the
|
||||
// current content. A lookup that fails for any other reason keeps the write
|
||||
// failure, so the event is retried.
|
||||
// described: it is deleted, it no longer references every chunk the event
|
||||
// named, or it references chunks the event did not. Dropped chunks are the ones
|
||||
// the filer deletes when an entry is updated, so they are gone from the volume
|
||||
// servers, or about to be, and no retry of the upload can succeed. Added chunks
|
||||
// mean the event was a snapshot of a write still in progress. A mount flushes
|
||||
// metadata while it is still uploading chunks in parallel, so such a snapshot
|
||||
// can have holes, and the stream reader stops at the first one: every retry
|
||||
// sends a short body and fails the same way. Chunks are compared by file id,
|
||||
// not content: a rewrite stores even identical bytes under new ids and drops
|
||||
// the old ones. A replay from an earlier offset (-timeAgo) re-emits such
|
||||
// events. Failing one holds the sync offset before it, so every restart of the
|
||||
// subscription replays it into the same failure, and progress on everything
|
||||
// after it in the log is never persisted. Skipping is safe: the event that
|
||||
// superseded this one follows in the log, and the delete removes the remote
|
||||
// object or the later write uploads the current content. A lookup that fails
|
||||
// for any other reason keeps the write failure, so the event is retried.
|
||||
func isSuperseded(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry) bool {
|
||||
current, _, _, err := filer_pb.GetEntry(context.Background(), filerClient, util.NewFullPath(dir, entry.Name))
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
@@ -965,12 +971,47 @@ func isSuperseded(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.
|
||||
if len(entry.Content) > 0 || len(current.Content) > 0 {
|
||||
return !bytes.Equal(entry.Content, current.Content)
|
||||
}
|
||||
return len(filer.DoMinusChunks(entry.GetChunks(), current.GetChunks())) > 0
|
||||
return len(filer.DoMinusChunks(entry.GetChunks(), current.GetChunks())) > 0 ||
|
||||
len(filer.DoMinusChunks(current.GetChunks(), entry.GetChunks())) > 0
|
||||
}
|
||||
|
||||
// errSuperseded marks an upload that failed for an entry the filer has since
|
||||
// moved past (isSuperseded). The caller skips the event instead of failing it.
|
||||
var errSuperseded = errors.New("deleted or rewritten since the event was logged")
|
||||
// hasReadHole reports an entry whose chunks do not cover its declared size: a
|
||||
// snapshot of a write still in progress, flushed while chunk uploads were
|
||||
// still running. The stream reader stops at the gap, so the upload fails on
|
||||
// every retry whether or not the filer ever moves past the entry — a write
|
||||
// that never finishes leaves the gap and wedges the event. Skipping is safe
|
||||
// either way: a finishing write logs its own event and uploads the full
|
||||
// content, and a dead one has nothing readable to lose.
|
||||
func hasReadHole(entry *filer_pb.Entry) bool {
|
||||
size := int64(entry.GetAttributes().GetFileSize())
|
||||
if size == 0 || len(entry.Content) > 0 {
|
||||
return false
|
||||
}
|
||||
chunks := slices.Clone(entry.GetChunks())
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
|
||||
return cmp.Compare(a.GetOffset(), b.GetOffset())
|
||||
})
|
||||
var next int64
|
||||
for _, chunk := range chunks {
|
||||
if chunk.GetOffset() >= size {
|
||||
break
|
||||
}
|
||||
if chunk.GetOffset() > next {
|
||||
return true
|
||||
}
|
||||
if end := chunk.GetOffset() + int64(chunk.GetSize()); end > next {
|
||||
next = end
|
||||
}
|
||||
}
|
||||
return next < size
|
||||
}
|
||||
|
||||
// errSuperseded marks an upload that cannot produce the entry an event
|
||||
// described: the filer moved past it (isSuperseded), or the entry is a
|
||||
// mid-write snapshot with a gap its chunks do not cover (hasReadHole). The
|
||||
// caller skips the event instead of failing it; a write completing logs a
|
||||
// later event that carries the full content.
|
||||
var errSuperseded = errors.New("superseded or incomplete since the event was logged")
|
||||
|
||||
// retriedWriteFile uploads the entry, retrying transient failures. The filer is
|
||||
// asked whether the entry is superseded before the first attempt and after
|
||||
@@ -981,6 +1022,9 @@ var errSuperseded = errors.New("deleted or rewritten since the event was logged"
|
||||
// still holds is worth sending. After: a dead chunk reads as a transient
|
||||
// "RequestError" from the SDK, and waiting out the backoff on it buys nothing.
|
||||
func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource filer_pb.FilerClient, dir string, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
|
||||
if hasReadHole(newEntry) {
|
||||
return nil, fmt.Errorf("%s %w", util.NewFullPath(dir, newEntry.Name), errSuperseded)
|
||||
}
|
||||
if isSuperseded(filerSource, dir, newEntry) {
|
||||
return nil, fmt.Errorf("%s %w", util.NewFullPath(dir, newEntry.Name), errSuperseded)
|
||||
}
|
||||
|
||||
@@ -517,6 +517,10 @@ func chunk(fileId, etag string) *filer_pb.FileChunk {
|
||||
return &filer_pb.FileChunk{FileId: fileId, Size: 1024, ETag: etag}
|
||||
}
|
||||
|
||||
func chunkAt(fileId, etag string, offset int64) *filer_pb.FileChunk {
|
||||
return &filer_pb.FileChunk{FileId: fileId, Offset: offset, Size: 1024, ETag: etag}
|
||||
}
|
||||
|
||||
func entryWith(name string, remote *filer_pb.RemoteEntry, chunks ...*filer_pb.FileChunk) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: remote, Chunks: chunks}
|
||||
}
|
||||
@@ -532,6 +536,11 @@ func entryWith(name string, remote *filer_pb.RemoteEntry, chunks ...*filer_pb.Fi
|
||||
// by chunk identity, the way the filer itself decides which chunks an update
|
||||
// leaves for deletion. Content fingerprints would miss a delete-and-recreate
|
||||
// of identical bytes and fail the event forever on its dead chunks.
|
||||
//
|
||||
// Chunks the filer has added since count too. The event was then a snapshot
|
||||
// of a write in progress: a mount flushes metadata while chunk uploads are
|
||||
// still running, so the snapshot can have holes, and the stream reader stops
|
||||
// at the first one on every retry.
|
||||
func TestIsSuperseded(t *testing.T) {
|
||||
const dir = "/buckets/ingest"
|
||||
event := entryWith("tmpjt9req69__Alan_Doe_Resume-1.pdf", nil, chunk("3,01", "e1"), chunk("3,02", "e2"))
|
||||
@@ -549,7 +558,7 @@ func TestIsSuperseded(t *testing.T) {
|
||||
{name: "truncated since", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"))}, want: true},
|
||||
{name: "still as described", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"))}, want: false},
|
||||
{name: "still as described, since replicated", filer: &stubFilerClient{entry: entryWith(event.Name, stamped, chunk("3,01", "e1"), chunk("3,02", "e2"))}, want: false},
|
||||
{name: "appended since, chunks still live", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"), chunk("3,03", "e3"))}, want: false},
|
||||
{name: "appended since", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"), chunk("3,03", "e3"))}, want: true},
|
||||
{name: "filer lookup failed", filer: &stubFilerClient{err: errors.New("rpc error: code = Unavailable")}, want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@@ -580,6 +589,17 @@ func TestIsSuperseded(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-write snapshot with a hole, filled in since", func(t *testing.T) {
|
||||
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
|
||||
filled := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("216,02", "e2", 1024), chunkAt("220,03", "e3", 2048))
|
||||
if !isSuperseded(&stubFilerClient{entry: filled}, dir, snapshot) {
|
||||
t.Error("isSuperseded = false, want true: the filer holds the finished write, the snapshot can never be read past its hole")
|
||||
}
|
||||
if isSuperseded(&stubFilerClient{entry: filled}, dir, filled) {
|
||||
t.Error("isSuperseded = true, want false for the event that describes the finished write")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inline content rewritten since", func(t *testing.T) {
|
||||
event := &filer_pb.Entry{Name: "note.txt", Content: []byte("v1")}
|
||||
if !isSuperseded(&stubFilerClient{entry: &filer_pb.Entry{Name: "note.txt", Content: []byte("v2")}}, dir, event) {
|
||||
@@ -652,6 +672,39 @@ func TestRetriedWriteFileStopsWhenSuperseded(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-write snapshot filled in during the attempt, one write", func(t *testing.T) {
|
||||
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
|
||||
filled := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("216,02", "e2", 1024), chunkAt("220,03", "e3", 2048))
|
||||
// What the S3 SDK reports when the stream reader stops at the hole.
|
||||
shortBody := errors.New("RequestError: send request failed\ncaused by: Put \"https://s3.example.com/tier/IMG_6526.mov?partNumber=12\": net/http: HTTP/1.x transport connection broken: http: ContentLength=8388608 with Body length 4194304")
|
||||
remote := &failingRemote{err: shortBody}
|
||||
filerClient := &supersedingFilerClient{live: snapshot, then: filled}
|
||||
_, err := retriedWriteFile(remote, filerClient, dir, snapshot, dest)
|
||||
if !errors.Is(err, errSuperseded) {
|
||||
t.Errorf("err = %v, want errSuperseded", err)
|
||||
}
|
||||
if remote.writes != 1 {
|
||||
t.Errorf("wrote %d times, want 1: the snapshot reads short on every retry", remote.writes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-write snapshot whose hole never fills, no write", func(t *testing.T) {
|
||||
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
|
||||
snapshot.Attributes.FileSize = 3072
|
||||
remote := &failingRemote{err: errors.New("RequestError: send request failed")}
|
||||
filerClient := &stubFilerClient{entry: snapshot}
|
||||
_, err := retriedWriteFile(remote, filerClient, dir, snapshot, dest)
|
||||
if !errors.Is(err, errSuperseded) {
|
||||
t.Errorf("err = %v, want errSuperseded", err)
|
||||
}
|
||||
if remote.writes != 0 {
|
||||
t.Errorf("wrote %d times, want 0: a snapshot with a hole reads short on every retry", remote.writes)
|
||||
}
|
||||
if filerClient.lookups != 0 {
|
||||
t.Errorf("looked up the filer %d times, want 0: no state makes this uploadable", filerClient.lookups)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("still as described, transient failure keeps retrying", func(t *testing.T) {
|
||||
prev := util.RetryWaitTime
|
||||
util.RetryWaitTime = 1600 * time.Millisecond // two attempts: 1s, then 1.5s
|
||||
@@ -993,6 +1046,7 @@ func TestIsFailedPrecondition(t *testing.T) {
|
||||
type supersedingFilerClient struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
live *filer_pb.Entry
|
||||
then *filer_pb.Entry // returned after the first lookup; nil means deleted
|
||||
lookups int
|
||||
}
|
||||
|
||||
@@ -1001,6 +1055,9 @@ func (c *supersedingFilerClient) LookupDirectoryEntry(context.Context, *filer_pb
|
||||
if c.lookups == 1 {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.live}, nil
|
||||
}
|
||||
if c.then != nil {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.then}, nil
|
||||
}
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user