Merge remote-tracking branch 'origin/master' into codex-rust-volume-server-bootstrap

# Conflicts:
#	.github/workflows/volume-server-integration-tests.yml
#	test/volume_server/framework/cluster.go
#	test/volume_server/framework/cluster_dual.go
#	test/volume_server/framework/volume_fixture.go
#	test/volume_server/grpc/data_stream_success_test.go
#	test/volume_server/http/public_cors_methods_test.go
#	test/volume_server/matrix/config_profiles.go

test/volume_server/http/admin_test.go

@@ -16,15 +16,13 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
 	client := framework.NewHTTPClient()
 
 	statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil)
 	if err != nil {
 		t.Fatalf("create status request: %v", err)
 	}
-	statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
-
 	statusResp := framework.DoRequest(t, client, statusReq)
 	statusBody := framework.ReadAllAndClose(t, statusResp)
 
@@ -34,8 +32,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
 		t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
 	}
-	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
-		t.Fatalf("expected echoed request id, got %q", got)
+	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
+		t.Fatal("expected server-generated request id in response header")
 	}
 
 	var payload map[string]interface{}
@@ -47,9 +45,21 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 			t.Fatalf("status payload missing field %q", field)
 		}
 	}
+	diskStatuses, ok := payload["DiskStatuses"].([]interface{})
+	if !ok || len(diskStatuses) == 0 {
+		t.Fatalf("status payload expected non-empty DiskStatuses, got %#v", payload["DiskStatuses"])
+	}
+	firstDisk, ok := diskStatuses[0].(map[string]interface{})
+	if !ok {
+		t.Fatalf("status payload disk status has unexpected shape: %#v", diskStatuses[0])
+	}
+	for _, field := range []string{"dir", "all", "used", "free"} {
+		if _, found := firstDisk[field]; !found {
+			t.Fatalf("status disk payload missing field %q: %#v", field, firstDisk)
+		}
+	}
 
 	healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
-	healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
 	healthResp := framework.DoRequest(t, client, healthReq)
 	_ = framework.ReadAllAndClose(t, healthResp)
 	if healthResp.StatusCode != http.StatusOK {
@@ -58,8 +68,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
 	if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
 		t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
 	}
-	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
-		t.Fatalf("expected /healthz echoed request id, got %q", got)
+	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
+		t.Fatal("expected /healthz server-generated request id in response header")
 	}
 
 	uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
@@ -77,7 +87,7 @@ func TestOptionsMethodsByPort(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
+	cluster := framework.StartVolumeCluster(t, matrix.P2())
 	client := framework.NewHTTPClient()
 
 	adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/"))
@@ -117,7 +127,7 @@ func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
+	cluster := framework.StartVolumeCluster(t, matrix.P2())
 	client := framework.NewHTTPClient()
 
 	adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")
@@ -154,7 +164,7 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P3())
+	cluster := framework.StartVolumeCluster(t, matrix.P3())
 	client := framework.NewHTTPClient()
 
 	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
@@ -170,7 +180,7 @@ func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	profile.AccessUI = true
+	profile.EnableUIAccess = true
 	cluster := framework.StartSingleVolumeCluster(t, profile)
 	client := framework.NewHTTPClient()
 

test/volume_server/http/auth_test.go

@@ -18,7 +18,7 @@ func TestJWTAuthForWriteAndRead(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -81,7 +81,7 @@ func TestJWTAuthRejectsFidMismatch(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -161,7 +161,7 @@ func TestJWTAuthRejectsExpiredTokens(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -209,7 +209,7 @@ func TestJWTAuthViaQueryParamAndCookie(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -249,7 +249,7 @@ func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -299,7 +299,7 @@ func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -345,7 +345,7 @@ func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) {
 	}
 
 	profile := matrix.P3()
-	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	clusterHarness := framework.StartVolumeCluster(t, profile)
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/chunk_manifest_test.go

@@ -16,7 +16,7 @@ func TestChunkManifestExpansionAndBypass(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -97,7 +97,7 @@ func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -174,7 +174,7 @@ func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/compressed_read_test.go

@@ -43,7 +43,7 @@ func TestCompressedReadAcceptEncodingMatrix(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/headers_static_test.go

@@ -15,7 +15,7 @@ func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -59,12 +59,50 @@ func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
 	}
 }
 
+func TestDownloadDispositionUsesGoBoolParsing(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(97)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fullFileID := framework.NewFileID(volumeID, 661123, 0x55667789)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("dl-bool-parse-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	parts := strings.SplitN(fullFileID, ",", 2)
+	if len(parts) != 2 {
+		t.Fatalf("unexpected file id format: %q", fullFileID)
+	}
+	fidOnly := parts[1]
+
+	url := fmt.Sprintf("%s/%d/%s/%s?dl=t", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "report.txt")
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("download read expected 200, got %d", resp.StatusCode)
+	}
+	contentDisposition := resp.Header.Get("Content-Disposition")
+	if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") {
+		t.Fatalf("download disposition with dl=t mismatch: %q", contentDisposition)
+	}
+}
+
 func TestStaticAssetEndpoints(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	client := framework.NewHTTPClient()
 
 	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico"))
@@ -85,7 +123,7 @@ func TestStaticAssetEndpointsOnPublicPort(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	client := framework.NewHTTPClient()
 
 	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico"))

test/volume_server/http/image_transform_test.go

@@ -45,7 +45,7 @@ func TestImageResizeAndCropReadVariants(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/production_features_test.go

@@ -0,0 +1,387 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestStatsEndpoints(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	// /stats/counter — expect 200 with non-empty body
+	// Note: Go server guards these with WhiteList which may return 400
+	counterResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/counter"))
+	counterBody := framework.ReadAllAndClose(t, counterResp)
+	if counterResp.StatusCode == http.StatusBadRequest {
+		t.Logf("/stats/counter returned 400 (whitelist guard), skipping stats checks")
+		return
+	}
+	if counterResp.StatusCode != http.StatusOK {
+		t.Fatalf("/stats/counter expected 200, got %d, body: %s", counterResp.StatusCode, string(counterBody))
+	}
+	if len(counterBody) == 0 {
+		t.Fatalf("/stats/counter returned empty body")
+	}
+
+	// /stats/memory — expect 200, valid JSON with Version and Memory
+	memoryResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/memory"))
+	memoryBody := framework.ReadAllAndClose(t, memoryResp)
+	if memoryResp.StatusCode != http.StatusOK {
+		t.Fatalf("/stats/memory expected 200, got %d, body: %s", memoryResp.StatusCode, string(memoryBody))
+	}
+	var memoryPayload map[string]any
+	if err := json.Unmarshal(memoryBody, &memoryPayload); err != nil {
+		t.Fatalf("/stats/memory response is not valid JSON: %v, body: %s", err, string(memoryBody))
+	}
+	if _, ok := memoryPayload["Version"]; !ok {
+		t.Fatalf("/stats/memory missing Version field")
+	}
+	if _, ok := memoryPayload["Memory"]; !ok {
+		t.Fatalf("/stats/memory missing Memory field")
+	}
+
+	// /stats/disk — expect 200, valid JSON with Version and DiskStatuses
+	diskResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/disk"))
+	diskBody := framework.ReadAllAndClose(t, diskResp)
+	if diskResp.StatusCode != http.StatusOK {
+		t.Fatalf("/stats/disk expected 200, got %d, body: %s", diskResp.StatusCode, string(diskBody))
+	}
+	var diskPayload map[string]any
+	if err := json.Unmarshal(diskBody, &diskPayload); err != nil {
+		t.Fatalf("/stats/disk response is not valid JSON: %v, body: %s", err, string(diskBody))
+	}
+	if _, ok := diskPayload["Version"]; !ok {
+		t.Fatalf("/stats/disk missing Version field")
+	}
+	if _, ok := diskPayload["DiskStatuses"]; !ok {
+		t.Fatalf("/stats/disk missing DiskStatuses field")
+	}
+}
+
+func TestStatusPrettyJsonAndCallbackIgnored(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	// ?pretty=y — expect indented multi-line JSON
+	prettyResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?pretty=y"))
+	prettyBody := framework.ReadAllAndClose(t, prettyResp)
+	if prettyResp.StatusCode != http.StatusOK {
+		t.Fatalf("/status?pretty=y expected 200, got %d", prettyResp.StatusCode)
+	}
+	lines := strings.Split(strings.TrimSpace(string(prettyBody)), "\n")
+	if len(lines) < 3 {
+		t.Fatalf("/status?pretty=y expected multi-line indented JSON, got %d lines: %s", len(lines), string(prettyBody))
+	}
+	var prettyPayload map[string]interface{}
+	if err := json.Unmarshal(prettyBody, &prettyPayload); err != nil {
+		t.Fatalf("/status?pretty=y is not valid JSON: %v", err)
+	}
+
+	// ?callback=myFunc — must be ignored; response is plain JSON with nosniff.
+	cbResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?callback=myFunc"))
+	cbBody := framework.ReadAllAndClose(t, cbResp)
+	if cbResp.StatusCode != http.StatusOK {
+		t.Fatalf("/status?callback=myFunc expected 200, got %d", cbResp.StatusCode)
+	}
+	if ct := cbResp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
+		t.Fatalf("/status?callback=myFunc expected Content-Type application/json, got %q", ct)
+	}
+	if nosniff := cbResp.Header.Get("X-Content-Type-Options"); nosniff != "nosniff" {
+		t.Fatalf("/status?callback=myFunc expected X-Content-Type-Options nosniff, got %q", nosniff)
+	}
+	if strings.Contains(string(cbBody), "myFunc(") {
+		t.Fatalf("/status?callback=myFunc must not wrap response in callback; body: %q", string(cbBody))
+	}
+	var cbPayload map[string]interface{}
+	if err := json.Unmarshal(cbBody, &cbPayload); err != nil {
+		t.Fatalf("/status?callback=myFunc is not valid JSON: %v", err)
+	}
+}
+
+func TestUploadWithCustomTimestamp(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(91)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 910001, 0xAABBCC01)
+	client := framework.NewHTTPClient()
+	data := []byte("custom-timestamp-data")
+
+	// Upload with ?ts=1700000000
+	uploadURL := fmt.Sprintf("%s/%s?ts=1700000000", cluster.VolumeAdminURL(), fid)
+	req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("create upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
+	uploadResp := framework.DoRequest(t, client, req)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload with ts expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	// Read back and verify Last-Modified
+	getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, getResp)
+	if getResp.StatusCode != http.StatusOK {
+		t.Fatalf("read expected 200, got %d", getResp.StatusCode)
+	}
+
+	expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
+	gotLastModified := getResp.Header.Get("Last-Modified")
+	if gotLastModified != expectedLastModified {
+		t.Fatalf("Last-Modified mismatch: got %q, want %q", gotLastModified, expectedLastModified)
+	}
+}
+
+func TestMultipartUploadUsesFormFieldsForTimestampAndTTL(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+	// Go's r.FormValue() cannot read multipart text fields after r.MultipartReader()
+	// consumes the body, so ts/ttl sent as multipart fields only work with the Rust server.
+	if os.Getenv("VOLUME_SERVER_IMPL") != "rust" {
+		t.Skip("skipping: multipart form field extraction for ts/ttl is Rust-only")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(94)
+	const needleID = uint64(940001)
+	const cookie = uint32(0xAABBCC04)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("multipart-form-fields-data")
+
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	if err := writer.WriteField("ts", "1700000000"); err != nil {
+		t.Fatalf("write multipart ts field: %v", err)
+	}
+	if err := writer.WriteField("ttl", "7d"); err != nil {
+		t.Fatalf("write multipart ttl field: %v", err)
+	}
+	filePart, err := writer.CreateFormFile("file", "multipart.txt")
+	if err != nil {
+		t.Fatalf("create multipart file field: %v", err)
+	}
+	if _, err := filePart.Write(payload); err != nil {
+		t.Fatalf("write multipart file payload: %v", err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatalf("close multipart writer: %v", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid, &body)
+	if err != nil {
+		t.Fatalf("create multipart upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	client := framework.NewHTTPClient()
+	uploadResp := framework.DoRequest(t, client, req)
+	uploadBody := framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("multipart upload expected 201, got %d, body: %s", uploadResp.StatusCode, string(uploadBody))
+	}
+
+	readResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("multipart upload read expected 200, got %d", readResp.StatusCode)
+	}
+	expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
+	if got := readResp.Header.Get("Last-Modified"); got != expectedLastModified {
+		t.Fatalf("multipart upload Last-Modified mismatch: got %q want %q", got, expectedLastModified)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus after multipart upload failed: %v", err)
+	}
+	// Go's ReadTTL normalizes via fitTtlCount: 7d → 1w (7 days = 1 week)
+	if got := statusResp.GetTtl(); got != "1w" {
+		t.Fatalf("multipart upload TTL mismatch: got %q want %q", got, "1w")
+	}
+}
+
+func TestRequestIdGeneration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	// GET /status WITHOUT setting x-amz-request-id header
+	req := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status")
+	resp := framework.DoRequest(t, client, req)
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("/status expected 200, got %d", resp.StatusCode)
+	}
+
+	reqID := resp.Header.Get("x-amz-request-id")
+	if reqID == "" {
+		t.Fatalf("expected auto-generated x-amz-request-id header, got empty")
+	}
+	// Go format: "%X%08X" (timestamp hex + 8 random hex), typically 20-24 chars, all hex, no hyphens.
+	if len(reqID) < 16 {
+		t.Fatalf("x-amz-request-id too short: %q (len=%d)", reqID, len(reqID))
+	}
+}
+
+func TestS3ResponsePassthroughHeaders(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(92)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 920001, 0xAABBCC02)
+	client := framework.NewHTTPClient()
+	data := []byte("passthrough-headers-test-data")
+
+	uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	// Read back with S3 passthrough query params
+	// Test response-content-language which both Go and Rust support
+	readURL := fmt.Sprintf("%s/%s?response-content-language=fr&response-expires=%s",
+		cluster.VolumeAdminURL(), fid,
+		"Thu,+01+Jan+2099+00:00:00+GMT",
+	)
+	readResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, readURL))
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("read with passthrough expected 200, got %d, body: %s", readResp.StatusCode, string(readBody))
+	}
+
+	if got := readResp.Header.Get("Content-Language"); got != "fr" {
+		t.Fatalf("Content-Language expected 'fr', got %q", got)
+	}
+	if got := readResp.Header.Get("Expires"); got != "Thu, 01 Jan 2099 00:00:00 GMT" {
+		t.Fatalf("Expires expected 'Thu, 01 Jan 2099 00:00:00 GMT', got %q", got)
+	}
+}
+
+func TestLargeFileWriteAndRead(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(93)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 930001, 0xAABBCC03)
+	client := framework.NewHTTPClient()
+	data := bytes.Repeat([]byte("A"), 1024*1024) // 1MB
+
+	uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload 1MB expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	getBody := framework.ReadAllAndClose(t, getResp)
+	if getResp.StatusCode != http.StatusOK {
+		t.Fatalf("read 1MB expected 200, got %d", getResp.StatusCode)
+	}
+	if len(getBody) != len(data) {
+		t.Fatalf("read 1MB body length mismatch: got %d, want %d", len(getBody), len(data))
+	}
+	if !bytes.Equal(getBody, data) {
+		t.Fatalf("read 1MB body content mismatch")
+	}
+}
+
+func TestUploadWithContentTypePreservation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(94)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 940001, 0xAABBCC04)
+	client := framework.NewHTTPClient()
+	data := []byte("fake-png-data-for-content-type-test")
+
+	// Upload with Content-Type: image/png
+	uploadURL := fmt.Sprintf("%s/%s", cluster.VolumeAdminURL(), fid)
+	req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("create upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "image/png")
+	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
+	uploadResp := framework.DoRequest(t, client, req)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload with image/png expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	// Read back and verify Content-Type is preserved
+	getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, getResp)
+	if getResp.StatusCode != http.StatusOK {
+		t.Fatalf("read expected 200, got %d", getResp.StatusCode)
+	}
+	if got := getResp.Header.Get("Content-Type"); got != "image/png" {
+		t.Fatalf("Content-Type expected 'image/png', got %q", got)
+	}
+}

test/volume_server/http/public_cors_methods_test.go

@@ -15,7 +15,7 @@ func TestPublicPortReadOnlyMethodBehavior(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -70,7 +70,7 @@ func TestCorsAndUnsupportedMethodBehavior(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -133,7 +133,7 @@ func TestUnsupportedMethodTraceParity(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -168,7 +168,7 @@ func TestUnsupportedMethodPropfindParity(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -207,12 +207,46 @@ func TestUnsupportedMethodPropfindParity(t *testing.T) {
 	}
 }
 
+func TestPublicPortHeadReadParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(86)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 124002, 0x04040404)
+	payload := []byte("public-head-parity-content")
+	client := framework.NewHTTPClient()
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
+	headBody := framework.ReadAllAndClose(t, headResp)
+	if headResp.StatusCode != http.StatusOK {
+		t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
+	}
+	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
+		t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
+	}
+	if len(headBody) != 0 {
+		t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
+	}
+}
 func TestUnsupportedMethodConnectParity(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -256,7 +290,7 @@ func TestUnsupportedMethodMkcolParity(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -294,38 +328,3 @@ func TestUnsupportedMethodMkcolParity(t *testing.T) {
 		t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
 	}
 }
-
-func TestPublicPortHeadReadParity(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test in short mode")
-	}
-
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
-	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
-	defer conn.Close()
-
-	const volumeID = uint32(86)
-	framework.AllocateVolume(t, grpcClient, volumeID, "")
-
-	fid := framework.NewFileID(volumeID, 124002, 0x04040404)
-	payload := []byte("public-head-parity-content")
-	client := framework.NewHTTPClient()
-
-	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
-	_ = framework.ReadAllAndClose(t, uploadResp)
-	if uploadResp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
-	}
-
-	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
-	headBody := framework.ReadAllAndClose(t, headResp)
-	if headResp.StatusCode != http.StatusOK {
-		t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
-	}
-	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
-		t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
-	}
-	if len(headBody) != 0 {
-		t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
-	}
-}

test/volume_server/http/range_variants_test.go

@@ -14,7 +14,7 @@ func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -52,7 +52,7 @@ func TestOversizedCombinedRangesAreIgnored(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/read_deleted_test.go

@@ -14,7 +14,7 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/read_path_variants_test.go

@@ -15,7 +15,7 @@ func TestReadPathShapesAndIfModifiedSince(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -85,7 +85,7 @@ func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	client := framework.NewHTTPClient()
 
 	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid"))
@@ -100,7 +100,7 @@ func TestReadWrongCookieReturnsNotFound(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -139,7 +139,7 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -174,11 +174,11 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
 	precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
 	precedenceResp := framework.DoRequest(t, client, precedenceReq)
 	precedenceBody := framework.ReadAllAndClose(t, precedenceResp)
-	if precedenceResp.StatusCode != http.StatusNotModified {
-		t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode)
+	if precedenceResp.StatusCode != http.StatusOK {
+		t.Fatalf("mismatched If-None-Match overrides If-Modified-Since: expected 200, got %d", precedenceResp.StatusCode)
 	}
-	if len(precedenceBody) != 0 {
-		t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody))
+	if string(precedenceBody) != string(payload) {
+		t.Fatalf("mismatched If-None-Match body mismatch: got %q want %q", string(precedenceBody), string(payload))
 	}
 
 	invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)

test/volume_server/http/read_write_delete_test.go

@@ -14,7 +14,7 @@ func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -112,7 +112,7 @@ func TestInvalidReadPathReturnsBadRequest(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
 	client := framework.NewHTTPClient()
 
 	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle"))

test/volume_server/http/replication_lifecycle_test.go

@@ -0,0 +1,102 @@
+package volume_server_http_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestReplicatedUploadSucceedsImmediatelyAfterAllocate(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const volumeID = uint32(115)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	req := &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Replication: "001",
+		Version:     uint32(needle.GetCurrentVersion()),
+	}
+	if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node0: %v", err)
+	}
+	if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node1: %v", err)
+	}
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 881001, 0x0B0C0D0E)
+	payload := []byte("replicated-upload-after-allocate")
+
+	// The master only learns about replica locations through volume-server
+	// heartbeats, which lag behind the direct AllocateVolume gRPC calls above.
+	// In production a client obtains its fid from the master assign flow, which
+	// guarantees the master already knows every replica; this test crafts the
+	// fid by hand, so the replicated write would otherwise look up the master
+	// before the second replica is registered and fail with a 500. Wait until
+	// the master reports both replicas before uploading.
+	if !waitForMasterReplicaCount(t, client, clusterHarness.MasterURL(), volumeID, 2, 10*time.Second) {
+		t.Fatalf("master did not report 2 replica locations for volume %d within deadline", volumeID)
+	}
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("replicated upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
+	var replicaBody []byte
+	if !waitForHTTPStatus(t, client, replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
+		replicaBody = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("replica did not become readable within deadline")
+	}
+	if string(replicaBody) != string(payload) {
+		t.Fatalf("replica body mismatch: got %q want %q", string(replicaBody), string(payload))
+	}
+}
+
+// waitForMasterReplicaCount polls the master volume lookup until it reports at
+// least want locations for volumeID, or the timeout elapses.
+func waitForMasterReplicaCount(t testing.TB, client *http.Client, masterURL string, volumeID uint32, want int, timeout time.Duration) bool {
+	t.Helper()
+
+	lookupURL := fmt.Sprintf("%s/dir/lookup?volumeId=%d", masterURL, volumeID)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, lookupURL))
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusOK {
+			var result struct {
+				Locations []struct {
+					Url string `json:"url"`
+				} `json:"locations"`
+			}
+			if err := json.Unmarshal(body, &result); err == nil && len(result.Locations) >= want {
+				return true
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	return false
+}

test/volume_server/http/throttling_test.go

@@ -60,7 +60,7 @@ func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -143,7 +143,7 @@ func TestUploadLimitWaitThenProceed(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -226,7 +226,7 @@ func TestUploadLimitTimeoutThenRecovery(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -289,7 +289,7 @@ func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -336,7 +336,7 @@ func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -398,7 +398,7 @@ func TestDownloadLimitTimeoutThenRecovery(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -593,7 +593,7 @@ func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -646,7 +646,7 @@ func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -691,7 +691,7 @@ func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/write_delete_variants_test.go

@@ -14,7 +14,7 @@ func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 
@@ -71,7 +71,7 @@ func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T)
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()
 

test/volume_server/http/write_error_variants_test.go

@@ -15,7 +15,7 @@ func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	client := framework.NewHTTPClient()
 
 	invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x"))
@@ -38,7 +38,7 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
 		t.Skip("skipping integration test in short mode")
 	}
 
-	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
 	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn.Close()