From f5989a21fb7c550a0a014a52bd664b10ecc3850d Mon Sep 17 00:00:00 2001 From: Nicholas Oh Date: Sat, 31 Oct 2026 20:28:54 +0800 Subject: [PATCH] s3: honor assignment fsync in streaming copies (#12277) --- .../s3api_object_handlers_copy_stream.go | 3 + .../s3api_object_handlers_copy_stream_test.go | 73 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 weed/s3api/s3api_object_handlers_copy_stream_test.go diff --git a/weed/s3api/s3api_object_handlers_copy_stream.go b/weed/s3api/s3api_object_handlers_copy_stream.go index 910ec3aca5..32a8babf90 100644 --- a/weed/s3api/s3api_object_handlers_copy_stream.go +++ b/weed/s3api/s3api_object_handlers_copy_stream.go @@ -102,6 +102,9 @@ func (s3a *S3ApiServer) streamCopyChunkRange( streamCtx, cancel := context.WithCancel(ctx) defer cancel() dstUrl := fmt.Sprintf("http://%s/%s", assignResult.Location.Url, assignResult.FileId) + if assignResult.Fsync { + dstUrl += "?fsync=true" + } dstJwt := security.EncodedJwt(assignResult.Auth) srcJwt := filer.JwtForVolumeServer(srcFileId) diff --git a/weed/s3api/s3api_object_handlers_copy_stream_test.go b/weed/s3api/s3api_object_handlers_copy_stream_test.go new file mode 100644 index 0000000000..9f0af11a40 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_copy_stream_test.go @@ -0,0 +1,73 @@ +package s3api + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" +) + +func TestStreamCopyChunkRangeFsync(t *testing.T) { + util_http.InitGlobalHttpClient() + payload := bytes.Repeat([]byte("copy data"), 512) + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer source.Close() + + for _, tc := range []struct { + name string + fsync bool + wantQuery string + status int + wantErr bool + }{ + {"without sync", false, "", http.StatusCreated, false}, + {"durable copy", true, "true", http.StatusCreated, false}, + {"durable write fails", true, "true", http.StatusInternalServerError, true}, + } { + t.Run(tc.name, func(t *testing.T) { + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("fsync"); got != tc.wantQuery { + t.Errorf("upload fsync=%q, want %q", got, tc.wantQuery) + } + reader, err := r.MultipartReader() + if err != nil { + t.Error(err) + http.Error(w, "invalid upload", http.StatusBadRequest) + return + } + part, err := reader.NextPart() + if err != nil { + t.Error(err) + http.Error(w, "missing data", http.StatusBadRequest) + return + } + data, err := io.ReadAll(part) + if err != nil || !bytes.Equal(data, payload) { + t.Errorf("copied payload differs: %v", err) + } + w.WriteHeader(tc.status) + _, _ = fmt.Fprintf(w, `{"size":%d}`, len(payload)) + })) + defer destination.Close() + assign := &filer_pb.AssignVolumeResponse{ + FileId: "1,test", + Location: &filer_pb.Location{Url: strings.TrimPrefix(destination.URL, "http://")}, + Fsync: tc.fsync, + } + err := (&S3ApiServer{}).streamCopyChunkRange(context.Background(), source.URL, + assign.FileId, 0, int64(len(payload)), true, assign, nil) + if (err != nil) != tc.wantErr { + t.Fatalf("stream copy error=%v, wantErr=%v", err, tc.wantErr) + } + }) + } +}