Compare commits

...
Author SHA1 Message Date
user.email 25896cf051 filer.remote.sync: skip a holed snapshot even while the filer still holds it
A mid-write snapshot the writer never finished is neither deleted nor
written past: the filer holds it unchanged, so the superseded check never
fires and every retry reads short at the same gap, pinning the sync offset
forever. The upload can never succeed for that entry — skip it like a
superseded one. A finishing write logs its own event and uploads the full
content; a dead write has nothing readable to lose.

The check also covers a chunkless entry that is not a dropped local copy
(e.g. a truncated remote-only entry): its reader yields zero bytes against
a nonzero size, which can silently clobber the remote object with an empty
body where the backend does not verify the length.
2026-10-28 10:44:17 -07:00
Alex Russell 8cf8abf9f6 filer.remote.sync: skip an event the filer has written past, not only rewritten
isSuperseded only looked for chunks the event named that the filer no
longer holds. A file copied in through a mount logs a create while the
copy is still running: the mount flushes metadata with chunk uploads in
flight, so that chunk list can have holes. ChunkStreamReader stops at
the first hole, the multipart upload sends a short part ("ContentLength=
8388608 with Body length 4194304"), and since every chunk the event
names is still live the event was never superseded. The daemon retried
it forever and the offset never moved past it.

Treat chunks the filer has added since the event the same way. The
event that added them follows in the log and uploads the finished file.
2026-10-28 18:04:04 +01:00
2 changed files with 118 additions and 17 deletions

No files matched your search

+60 -16
View File
@@ -2,11 +2,13 @@ package command
import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"io"
"os"
"slices"
"strings"
"time"
@@ -936,18 +938,22 @@ func readRemoteStorageConf(filerClient filer_pb.FilerClient, storageName string)
}
// isSuperseded reports whether the filer has moved past the entry an event
// described: it is deleted, or it no longer references every chunk the event
// named. Those are the chunks the filer deletes when an entry is updated, so
// they are gone from the volume servers, or about to be, and no retry of the
// upload can succeed. Chunks are compared by file id, not content: a rewrite
// stores even identical bytes under new ids and drops the old ones. A replay
// from an earlier offset (-timeAgo) re-emits such events. Failing one holds the
// sync offset before it, so every restart of the subscription replays it into
// the same dead chunks, and progress on everything after it in the log is never
// persisted. Skipping is safe: the event that superseded this one follows in
// the log, and the delete removes the remote object or the rewrite uploads the
// current content. A lookup that fails for any other reason keeps the write
// failure, so the event is retried.
// described: it is deleted, it no longer references every chunk the event
// named, or it references chunks the event did not. Dropped chunks are the ones
// the filer deletes when an entry is updated, so they are gone from the volume
// servers, or about to be, and no retry of the upload can succeed. Added chunks
// mean the event was a snapshot of a write still in progress. A mount flushes
// metadata while it is still uploading chunks in parallel, so such a snapshot
// can have holes, and the stream reader stops at the first one: every retry
// sends a short body and fails the same way. Chunks are compared by file id,
// not content: a rewrite stores even identical bytes under new ids and drops
// the old ones. A replay from an earlier offset (-timeAgo) re-emits such
// events. Failing one holds the sync offset before it, so every restart of the
// subscription replays it into the same failure, and progress on everything
// after it in the log is never persisted. Skipping is safe: the event that
// superseded this one follows in the log, and the delete removes the remote
// object or the later write uploads the current content. A lookup that fails
// for any other reason keeps the write failure, so the event is retried.
func isSuperseded(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry) bool {
current, _, _, err := filer_pb.GetEntry(context.Background(), filerClient, util.NewFullPath(dir, entry.Name))
if errors.Is(err, filer_pb.ErrNotFound) {
@@ -965,12 +971,47 @@ func isSuperseded(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.
if len(entry.Content) > 0 || len(current.Content) > 0 {
return !bytes.Equal(entry.Content, current.Content)
}
return len(filer.DoMinusChunks(entry.GetChunks(), current.GetChunks())) > 0
return len(filer.DoMinusChunks(entry.GetChunks(), current.GetChunks())) > 0 ||
len(filer.DoMinusChunks(current.GetChunks(), entry.GetChunks())) > 0
}
// errSuperseded marks an upload that failed for an entry the filer has since
// moved past (isSuperseded). The caller skips the event instead of failing it.
var errSuperseded = errors.New("deleted or rewritten since the event was logged")
// hasReadHole reports an entry whose chunks do not cover its declared size: a
// snapshot of a write still in progress, flushed while chunk uploads were
// still running. The stream reader stops at the gap, so the upload fails on
// every retry whether or not the filer ever moves past the entry — a write
// that never finishes leaves the gap and wedges the event. Skipping is safe
// either way: a finishing write logs its own event and uploads the full
// content, and a dead one has nothing readable to lose.
func hasReadHole(entry *filer_pb.Entry) bool {
size := int64(entry.GetAttributes().GetFileSize())
if size == 0 || len(entry.Content) > 0 {
return false
}
chunks := slices.Clone(entry.GetChunks())
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
return cmp.Compare(a.GetOffset(), b.GetOffset())
})
var next int64
for _, chunk := range chunks {
if chunk.GetOffset() >= size {
break
}
if chunk.GetOffset() > next {
return true
}
if end := chunk.GetOffset() + int64(chunk.GetSize()); end > next {
next = end
}
}
return next < size
}
// errSuperseded marks an upload that cannot produce the entry an event
// described: the filer moved past it (isSuperseded), or the entry is a
// mid-write snapshot with a gap its chunks do not cover (hasReadHole). The
// caller skips the event instead of failing it; a write completing logs a
// later event that carries the full content.
var errSuperseded = errors.New("superseded or incomplete since the event was logged")
// retriedWriteFile uploads the entry, retrying transient failures. The filer is
// asked whether the entry is superseded before the first attempt and after
@@ -981,6 +1022,9 @@ var errSuperseded = errors.New("deleted or rewritten since the event was logged"
// still holds is worth sending. After: a dead chunk reads as a transient
// "RequestError" from the SDK, and waiting out the backoff on it buys nothing.
func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource filer_pb.FilerClient, dir string, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
if hasReadHole(newEntry) {
return nil, fmt.Errorf("%s %w", util.NewFullPath(dir, newEntry.Name), errSuperseded)
}
if isSuperseded(filerSource, dir, newEntry) {
return nil, fmt.Errorf("%s %w", util.NewFullPath(dir, newEntry.Name), errSuperseded)
}
+58 -1
View File
@@ -517,6 +517,10 @@ func chunk(fileId, etag string) *filer_pb.FileChunk {
return &filer_pb.FileChunk{FileId: fileId, Size: 1024, ETag: etag}
}
func chunkAt(fileId, etag string, offset int64) *filer_pb.FileChunk {
return &filer_pb.FileChunk{FileId: fileId, Offset: offset, Size: 1024, ETag: etag}
}
func entryWith(name string, remote *filer_pb.RemoteEntry, chunks ...*filer_pb.FileChunk) *filer_pb.Entry {
return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: remote, Chunks: chunks}
}
@@ -532,6 +536,11 @@ func entryWith(name string, remote *filer_pb.RemoteEntry, chunks ...*filer_pb.Fi
// by chunk identity, the way the filer itself decides which chunks an update
// leaves for deletion. Content fingerprints would miss a delete-and-recreate
// of identical bytes and fail the event forever on its dead chunks.
//
// Chunks the filer has added since count too. The event was then a snapshot
// of a write in progress: a mount flushes metadata while chunk uploads are
// still running, so the snapshot can have holes, and the stream reader stops
// at the first one on every retry.
func TestIsSuperseded(t *testing.T) {
const dir = "/buckets/ingest"
event := entryWith("tmpjt9req69__Alan_Doe_Resume-1.pdf", nil, chunk("3,01", "e1"), chunk("3,02", "e2"))
@@ -549,7 +558,7 @@ func TestIsSuperseded(t *testing.T) {
{name: "truncated since", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"))}, want: true},
{name: "still as described", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"))}, want: false},
{name: "still as described, since replicated", filer: &stubFilerClient{entry: entryWith(event.Name, stamped, chunk("3,01", "e1"), chunk("3,02", "e2"))}, want: false},
{name: "appended since, chunks still live", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"), chunk("3,03", "e3"))}, want: false},
{name: "appended since", filer: &stubFilerClient{entry: entryWith(event.Name, nil, chunk("3,01", "e1"), chunk("3,02", "e2"), chunk("3,03", "e3"))}, want: true},
{name: "filer lookup failed", filer: &stubFilerClient{err: errors.New("rpc error: code = Unavailable")}, want: false},
}
for _, tt := range tests {
@@ -580,6 +589,17 @@ func TestIsSuperseded(t *testing.T) {
}
})
t.Run("mid-write snapshot with a hole, filled in since", func(t *testing.T) {
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
filled := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("216,02", "e2", 1024), chunkAt("220,03", "e3", 2048))
if !isSuperseded(&stubFilerClient{entry: filled}, dir, snapshot) {
t.Error("isSuperseded = false, want true: the filer holds the finished write, the snapshot can never be read past its hole")
}
if isSuperseded(&stubFilerClient{entry: filled}, dir, filled) {
t.Error("isSuperseded = true, want false for the event that describes the finished write")
}
})
t.Run("inline content rewritten since", func(t *testing.T) {
event := &filer_pb.Entry{Name: "note.txt", Content: []byte("v1")}
if !isSuperseded(&stubFilerClient{entry: &filer_pb.Entry{Name: "note.txt", Content: []byte("v2")}}, dir, event) {
@@ -652,6 +672,39 @@ func TestRetriedWriteFileStopsWhenSuperseded(t *testing.T) {
}
})
t.Run("mid-write snapshot filled in during the attempt, one write", func(t *testing.T) {
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
filled := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("216,02", "e2", 1024), chunkAt("220,03", "e3", 2048))
// What the S3 SDK reports when the stream reader stops at the hole.
shortBody := errors.New("RequestError: send request failed\ncaused by: Put \"https://s3.example.com/tier/IMG_6526.mov?partNumber=12\": net/http: HTTP/1.x transport connection broken: http: ContentLength=8388608 with Body length 4194304")
remote := &failingRemote{err: shortBody}
filerClient := &supersedingFilerClient{live: snapshot, then: filled}
_, err := retriedWriteFile(remote, filerClient, dir, snapshot, dest)
if !errors.Is(err, errSuperseded) {
t.Errorf("err = %v, want errSuperseded", err)
}
if remote.writes != 1 {
t.Errorf("wrote %d times, want 1: the snapshot reads short on every retry", remote.writes)
}
})
t.Run("mid-write snapshot whose hole never fills, no write", func(t *testing.T) {
snapshot := entryWith("IMG_6526.mov", nil, chunkAt("222,01", "e1", 0), chunkAt("220,03", "e3", 2048))
snapshot.Attributes.FileSize = 3072
remote := &failingRemote{err: errors.New("RequestError: send request failed")}
filerClient := &stubFilerClient{entry: snapshot}
_, err := retriedWriteFile(remote, filerClient, dir, snapshot, dest)
if !errors.Is(err, errSuperseded) {
t.Errorf("err = %v, want errSuperseded", err)
}
if remote.writes != 0 {
t.Errorf("wrote %d times, want 0: a snapshot with a hole reads short on every retry", remote.writes)
}
if filerClient.lookups != 0 {
t.Errorf("looked up the filer %d times, want 0: no state makes this uploadable", filerClient.lookups)
}
})
t.Run("still as described, transient failure keeps retrying", func(t *testing.T) {
prev := util.RetryWaitTime
util.RetryWaitTime = 1600 * time.Millisecond // two attempts: 1s, then 1.5s
@@ -993,6 +1046,7 @@ func TestIsFailedPrecondition(t *testing.T) {
type supersedingFilerClient struct {
filer_pb.SeaweedFilerClient
live *filer_pb.Entry
then *filer_pb.Entry // returned after the first lookup; nil means deleted
lookups int
}
@@ -1001,6 +1055,9 @@ func (c *supersedingFilerClient) LookupDirectoryEntry(context.Context, *filer_pb
if c.lookups == 1 {
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.live}, nil
}
if c.then != nil {
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.then}, nil
}
return nil, filer_pb.ErrNotFound
}