filer.sync: shorten retries for confirmed missing chunks (#12546)

* filer.sync: shorten retries for confirmed missing chunks

* filer.sync: clarify missing chunk grace period names

weed/replication/source/filer_source.go

@@ -25,6 +25,10 @@ import (
 // still get past.
 var ErrVolumeNotFound = errors.New("volume not found")
 
+// ErrChunkNotFound reports that every located volume replica answered 404 for
+// the chunk. A failed request or a filer-proxied 404 cannot establish this.
+var ErrChunkNotFound = errors.New("chunk not found on any volume replica")
+
 type FilerSource struct {
 	grpcAddress    string
 	grpcDialOption grpc.DialOption
@@ -165,6 +169,7 @@ func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, h
 		return "", nil, nil, err
 	}
 
+	var replicaErr error
 	for _, fileUrl := range fileUrls {
 		filename, header, resp, err = downloadFn(fileUrl, "", offset)
 		if err == nil {
@@ -172,12 +177,24 @@ func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, h
 		}
 		if err != nil {
 			resp = nil
+			if !errors.Is(err, util_http.ErrNotFound) {
+				replicaErr = err
+			}
 			glog.V(0).Infof("fail to read part %s from %s offset %d: %v", fileId, fileUrl, offset, err)
 			continue
 		}
 		glog.V(4).Infof("read part %s from %s offset %d content-length:%s", fileId, fileUrl, offset, header.Get("Content-Length"))
 		break
 	}
+	if err != nil {
+		if replicaErr != nil {
+			// An unreachable replica may still hold the chunk. Its failure must
+			// not be hidden by another replica answering 404 later in the list.
+			err = replicaErr
+		} else if errors.Is(err, util_http.ErrNotFound) {
+			err = fmt.Errorf("%w: %w", ErrChunkNotFound, err)
+		}
+	}
 
 	return filename, header, resp, err
 }

weed/replication/source/filer_source_test.go

@@ -378,18 +378,90 @@ func TestReadPartSkipsNotFoundReplica(t *testing.T) {
 // Every replica gone: the caller must see a not-found error it can act on, and no
 // response left open behind it.
 func TestReadPartAllReplicasNotFound(t *testing.T) {
-	gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	var hits atomic.Int32
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
 		http.Error(w, "Not Found", http.StatusNotFound)
-	}))
+	})
+	gone := httptest.NewServer(handler)
 	defer gone.Close()
+	second := httptest.NewServer(handler)
+	defer second.Close()
 
-	filerSource := startLookupFiler(t, strings.TrimPrefix(gone.URL, "http://"))
+	filerSource := startLookupFiler(t, strings.TrimPrefix(gone.URL, "http://"), strings.TrimPrefix(second.URL, "http://"))
 
 	_, _, resp, err := filerSource.ReadPart("5617,01abc", 0)
 	if !errors.Is(err, util_http.ErrNotFound) {
 		t.Fatalf("expected ErrNotFound, got %v", err)
 	}
+	if !errors.Is(err, ErrChunkNotFound) {
+		t.Fatalf("all replicas answered 404, expected ErrChunkNotFound, got %v", err)
+	}
+	if hits.Load() != 2 {
+		t.Fatalf("expected every replica to be tried, got %d requests", hits.Load())
+	}
 	if resp != nil {
 		t.Fatal("expected no response alongside the error")
 	}
 }
+
+// A 404 from one replica cannot rule out a chunk held by another replica that
+// failed to answer. Preserve the other failure regardless of replica order.
+func TestReadPartPreservesReplicaFailure(t *testing.T) {
+	gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Not Found", http.StatusNotFound)
+	}))
+	defer gone.Close()
+	for _, tc := range []struct {
+		name        string
+		status      int
+		unreachable bool
+	}{
+		{http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable, false},
+		{http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized, false},
+		{"connection refused", http.StatusServiceUnavailable, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			failed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(tc.status)
+			}))
+			defer failed.Close()
+			if tc.unreachable {
+				failed.Close()
+			}
+			locations := []string{strings.TrimPrefix(failed.URL, "http://"), strings.TrimPrefix(gone.URL, "http://")}
+			for _, reverse := range []bool{false, true} {
+				if reverse {
+					locations[0], locations[1] = locations[1], locations[0]
+				}
+				filerSource := startLookupFiler(t, locations...)
+				_, _, resp, err := filerSource.ReadPart("5617,01abc", 0)
+				if err == nil || !strings.Contains(err.Error(), tc.name) {
+					t.Fatalf("expected replica failure %s, got %v", tc.name, err)
+				}
+				if errors.Is(err, ErrChunkNotFound) || errors.Is(err, util_http.ErrNotFound) {
+					t.Fatalf("a failed replica must not count as a missing chunk: %v", err)
+				}
+				if resp != nil {
+					t.Fatal("expected no response alongside the error")
+				}
+			}
+		})
+	}
+}
+
+func TestReadPartProxyNotFoundIsUnconfirmed(t *testing.T) {
+	gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Not Found", http.StatusNotFound)
+	}))
+	defer gone.Close()
+	address := strings.TrimPrefix(gone.URL, "http://")
+	filerSource := &FilerSource{}
+	if err := filerSource.DoInitialize(address, address, "/", true); err != nil {
+		t.Fatal(err)
+	}
+	_, _, _, err := filerSource.ReadPart("5617,01abc", 0)
+	if !errors.Is(err, util_http.ErrNotFound) || errors.Is(err, ErrChunkNotFound) {
+		t.Fatalf("a proxied 404 must remain ambiguous: %v", err)
+	}
+}