mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-15 11:00:51 +02:00
Konstantin AdzerandChris Lu
42b0ca7850
* s3 sink: report the source read error the SDK hides filer.backup stops for good on an event whose chunks are gone from the volume servers: the uploader reads the body, the read fails with the volume's 404, and the AWS SDK returns "ContentLength=N with Body length 0" without the cause. isIgnorable404 would skip such an event, but it never sees the 404, so the event is retried forever and the checkpoint never advances. ChunkStreamReader keeps its first source failure and the s3 sink returns it when the upload fails. * s3 sink: trim verbose comments on source error propagation --------- Co-authored-by: Chris Lu <chris.lu@gmail.com>
56 lines
1.6 KiB
Go
56 lines
1.6 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
func TestChunkStreamReaderSourceError(t *testing.T) {
|
|
server := createTestServer(map[string][]byte{})
|
|
defer server.Close()
|
|
|
|
lookup := func(ctx context.Context, fileId string) ([]string, error) {
|
|
return []string{server.URL + "/" + fileId}, nil
|
|
}
|
|
reader := NewChunkStreamReaderFromLookup(context.Background(), lookup,
|
|
[]*filer_pb.FileChunk{{FileId: "7,01637037d6", Size: 8}})
|
|
|
|
if _, err := io.ReadAll(reader); err == nil {
|
|
t.Fatal("reading a gone chunk succeeded")
|
|
}
|
|
sourceErr := reader.SourceError()
|
|
if sourceErr == nil {
|
|
t.Fatal("reader kept no source error")
|
|
}
|
|
if !strings.Contains(sourceErr.Error(), "404") {
|
|
t.Fatalf("source error does not name the volume answer: %v", sourceErr)
|
|
}
|
|
if ReaderSourceError(reader) != sourceErr {
|
|
t.Fatal("ReaderSourceError does not return the kept error")
|
|
}
|
|
}
|
|
|
|
func TestChunkStreamReaderSourceErrorOnLookup(t *testing.T) {
|
|
master := &testMasterClient{urls: map[string][]string{}}
|
|
reader := NewChunkStreamReaderFromLookup(context.Background(), master.GetLookupFileIdFunction(),
|
|
[]*filer_pb.FileChunk{{FileId: "7,01637037d6", Size: 8}})
|
|
|
|
if _, err := io.ReadAll(reader); err == nil {
|
|
t.Fatal("reading a chunk with no location succeeded")
|
|
}
|
|
if reader.SourceError() == nil {
|
|
t.Fatal("reader kept no source error")
|
|
}
|
|
}
|
|
|
|
func TestReaderSourceErrorInlineEntry(t *testing.T) {
|
|
reader := NewFileReader(nil, &filer_pb.Entry{Content: []byte("inline")})
|
|
if err := ReaderSourceError(reader); err != nil {
|
|
t.Fatalf("inline content reported a source error: %v", err)
|
|
}
|
|
}
|