filer sync: do not advance the sync offset past a failed event (#10424)

* util: retry transient errors, not just the ones containing "transport"

util.Retry only retried when the error string contained "transport", so a
plain "read: connection reset by peer" from S3 got zero retries. Classify
the error instead: net timeouts, connection resets, and the throttling and
overload replies S3 and gRPC return are all worth another attempt, while a
cancelled or expired context is not.

* filer sync: hold the sync offset behind a failed event

A sync job that returned an error was logged and forgotten, and the
watermark advanced past it anyway. The offset is the durable resume point,
so the event was never replayed: for filer.remote.sync that left the file
present locally, absent on the remote, with no RemoteEntry and nothing to
retry it.

Pin the watermark at the oldest failed event. Later events keep flowing,
but the persisted offset stays behind the failure, so a restart replays it.

weed/command/filer_sync_jobs.go

@@ -5,6 +5,7 @@ import (
 	"path"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -86,6 +87,12 @@ type MetadataProcessor struct {
 	// tsHeap is a min-heap of active job timestamps with lazy deletion,
 	// used for O(log n) amortized watermark tracking.
 	tsHeap tsMinHeap
+
+	// oldestFailedTsNs is the timestamp of the oldest event whose job returned
+	// an error, or 0 when none has. The watermark is never advanced to it or
+	// past it, so the persisted sync offset stays behind the failure and a
+	// restart replays the event instead of skipping it forever.
+	oldestFailedTsNs int64
 }
 
 func NewMetadataProcessor(fn pb.ProcessMetadataFunc, concurrency int, offsetTsNs int64) *MetadataProcessor {
@@ -254,15 +261,22 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse)
 
 	go func() {
 
-		if err := util.Retry("metadata processor", func() error {
+		jobErr := util.Retry("metadata processor", func() error {
 			return t.fn(resp)
-		}); err != nil {
-			glog.Errorf("process %v: %v", resp, err)
-		}
+		})
 
 		t.activeJobsLock.Lock()
 		defer t.activeJobsLock.Unlock()
 
+		if jobErr != nil {
+			if t.oldestFailedTsNs == 0 || resp.TsNs < t.oldestFailedTsNs {
+				t.oldestFailedTsNs = resp.TsNs
+				glog.Errorf("process %v: %v; holding sync offset at %v so this event is replayed on restart", resp, jobErr, time.Unix(0, resp.TsNs))
+			} else {
+				glog.Errorf("process %v: %v", resp, jobErr)
+			}
+		}
+
 		delete(t.activeJobs, resp.TsNs)
 		t.removePathFromIndex(jobPaths.path, jobPaths.kind)
 		if jobPaths.newPath != "" {
@@ -277,9 +291,13 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse)
 			}
 			heap.Pop(&t.tsHeap)
 		}
-		// If this was the oldest job, advance the watermark.
+		// If this was the oldest job, advance the watermark, but never to or
+		// past an event that failed: the offset is the durable resume point,
+		// and moving it over a failure drops that event for good.
 		if t.tsHeap.Len() == 0 || resp.TsNs < t.tsHeap[0] {
-			t.processedTsWatermark.Store(resp.TsNs)
+			if t.oldestFailedTsNs == 0 || resp.TsNs < t.oldestFailedTsNs {
+				t.processedTsWatermark.Store(resp.TsNs)
+			}
 		}
 		t.activeJobsCond.Signal()
 	}()

weed/command/filer_sync_jobs_test.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"container/heap"
+	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -610,3 +611,76 @@ func TestMetadataProcessorEmptyMarkerKeepsWatermarkStale(t *testing.T) {
 	}
 	t.Logf("marker carried fresh ts %d but watermark stayed stale at %d", freshTs, staleOffset)
 }
+
+// waitForJobsToDrain blocks until every job goroutine has finished bookkeeping.
+func waitForJobsToDrain(t *testing.T, p *MetadataProcessor) {
+	t.Helper()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		p.activeJobsLock.Lock()
+		remaining := len(p.activeJobs)
+		p.activeJobsLock.Unlock()
+		if remaining == 0 {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatal("timed out waiting for sync jobs to drain")
+}
+
+// TestFailedJobHoldsWatermark verifies that a job that returns an error keeps
+// the watermark — and therefore the persisted sync offset — behind the failed
+// event, so a restart replays it. Advancing past it drops the event for good:
+// the file stays local-only and nothing ever retries the upload.
+func TestFailedJobHoldsWatermark(t *testing.T) {
+	const failedTsNs = int64(200)
+	// a permanent error, so util.Retry gives up on the first attempt
+	fn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if resp.TsNs == failedTsNs {
+			return errors.New("AccessDenied: Access Denied")
+		}
+		return nil
+	}
+	// concurrency 1 runs the jobs serially in timestamp order
+	p := NewMetadataProcessor(fn, 1, 0)
+
+	p.AddSyncJob(makeResp("/dir", "a.txt", false, 100, true))
+	waitForJobsToDrain(t, p)
+	if got := p.processedTsWatermark.Load(); got != 100 {
+		t.Fatalf("watermark = %d after a successful job, want 100", got)
+	}
+
+	p.AddSyncJob(makeResp("/dir", "b.txt", false, failedTsNs, true))
+	waitForJobsToDrain(t, p)
+	if got := p.processedTsWatermark.Load(); got != 100 {
+		t.Fatalf("watermark = %d after a failed job, want it held at 100", got)
+	}
+
+	// later events keep flowing, but the offset stays behind the failure
+	p.AddSyncJob(makeResp("/dir", "c.txt", false, 300, true))
+	waitForJobsToDrain(t, p)
+	if got := p.processedTsWatermark.Load(); got != 100 {
+		t.Fatalf("watermark = %d after a later success, want it held at 100", got)
+	}
+}
+
+// TestFailedJobHoldsWatermarkAtOldestFailure verifies that the watermark is
+// pinned by the oldest failure, not the most recent one.
+func TestFailedJobHoldsWatermarkAtOldestFailure(t *testing.T) {
+	fn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		if resp.TsNs == 200 || resp.TsNs == 400 {
+			return errors.New("AccessDenied: Access Denied")
+		}
+		return nil
+	}
+	p := NewMetadataProcessor(fn, 1, 0)
+
+	for _, ts := range []int64{100, 200, 300, 400, 500} {
+		p.AddSyncJob(makeResp("/dir", fmt.Sprintf("f%d.txt", ts), false, ts, true))
+		waitForJobsToDrain(t, p)
+	}
+
+	if got := p.processedTsWatermark.Load(); got != 100 {
+		t.Fatalf("watermark = %d, want it held at 100 by the failure at 200", got)
+	}
+}