From 9e5357782e5f807aee40ac957b57c3ab76b7f9bb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 16 Nov 2026 16:20:15 -0800 Subject: [PATCH] volume: keep reads over the download limit local when readMode is local (#12679) * volume: keep reads over the download limit local when readMode is local With -readMode=local the server never sends a read for a volume it does not hold elsewhere, but a read over -concurrentDownloadLimitMB on a replicated volume was still redirected to another replica. Skip the replica fallback in local mode, so the read waits for a download slot and answers 429 when the wait times out. Co-Authored-By: Claude Opus 5.5 * volume(rs): keep reads over the download limit local when readMode is local Mirror the Go volume server: with readMode local, a read over the download limit waits for a slot instead of being redirected to a replica. proxy_or_redirect_to_target no longer needs the local-mode redirect switch. Co-Authored-By: Claude Opus 5.5 * volume: check download capacity under the wait lock waitForDownloadSlot read the in-flight size outside the cond lock and the read handler broadcast without it, so a download finishing between the check and the wait was missed and the read waited out the timeout for a 429. Check capacity with util.WaitUntil under the lock and broadcast under it. Co-Authored-By: Claude Opus 5.5 * volume(rs): register for the download notification before checking capacity The wait loop created its Notified future only after the capacity check, so a body dropped in between had its notify_waiters() lost and the read waited out the deadline for a 429. Create the future first, then check, as Go now checks under the cond lock. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/config.rs | 3 +- seaweed-volume/src/server/handlers.rs | 111 +++++++++++++++--- weed/command/mini.go | 2 +- weed/command/server.go | 2 +- weed/command/volume.go | 2 +- weed/server/volume_server_handlers.go | 48 ++++---- ...ume_server_handlers_download_limit_test.go | 87 ++++++++++++++ weed/server/volume_server_handlers_read.go | 2 + 8 files changed, 215 insertions(+), 42 deletions(-) create mode 100644 weed/server/volume_server_handlers_download_limit_test.go diff --git a/seaweed-volume/src/config.rs b/seaweed-volume/src/config.rs index 58efcf8adb..c1a02b2b18 100644 --- a/seaweed-volume/src/config.rs +++ b/seaweed-volume/src/config.rs @@ -138,7 +138,8 @@ pub struct Cli { #[arg(long = "concurrentUploadLimitMB", default_value_t = 0)] pub concurrent_upload_limit_mb: u32, - /// Limit total concurrent download size in MB, 0 means unlimited. + /// Limit total concurrent download size in MB, 0 means unlimited; reads over + /// the limit are sent to a replica unless readMode is local. #[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)] pub concurrent_download_limit_mb: u32, diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 883b296b8f..9b1d4b21ed 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -913,7 +913,6 @@ async fn proxy_or_redirect_to_target( state: &VolumeServerState, info: ProxyRequestInfo, vid: VolumeId, - allow_local_redirect: bool, ) -> Response { // Look up volume locations from master let locations = match lookup_volume( @@ -961,10 +960,7 @@ async fn proxy_or_redirect_to_target( match state.read_mode { ReadMode::Proxy => proxy_request(state, &info, target).await, ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme), - ReadMode::Local if allow_local_redirect => { - redirect_request(&info, target, &state.outgoing_http_scheme) - } - ReadMode::Local => unreachable!(), + ReadMode::Local => StatusCode::NOT_FOUND.into_response(), } } @@ -1431,7 +1427,7 @@ async fn proxy_missing_volume( None => return StatusCode::NOT_FOUND.into_response(), }; - proxy_or_redirect_to_target(state, info, vid, false).await + proxy_or_redirect_to_target(state, info, vid).await } /// Go's `proxied` query param check (loop prevention). @@ -1464,8 +1460,10 @@ async fn check_download_limit( // Go tries proxy to replica once before the wait loop // (checkDownloadLimit); it does not retry on each wakeup. - let should_try_replica = - !is_proxied_query(&query_string) && !state.current_master().await.is_empty() && { + let should_try_replica = state.read_mode != ReadMode::Local + && !is_proxied_query(&query_string) + && !state.current_master().await.is_empty() + && { let store = state.store.read().unwrap(); store .find_volume(vid) @@ -1475,25 +1473,24 @@ async fn check_download_limit( && let Some(info) = build_proxy_request_info(path, request_headers, &query_string, method) { - return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await); + return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await); } // Blocking wait loop (Go's waitForDownloadSlot) loop { - if tokio::time::timeout_at(deadline, state.download_notify.notified()) - .await - .is_err() + let notified = state.download_notify.notified(); + if state.inflight_download_bytes.load(Ordering::Relaxed) + <= state.concurrent_download_limit { + break; + } + if tokio::time::timeout_at(deadline, notified).await.is_err() { return ControlFlow::Break(json_error_with_query( StatusCode::TOO_MANY_REQUESTS, "download limit exceeded", uri.query(), )); } - let current = state.inflight_download_bytes.load(Ordering::Relaxed); - if current <= state.concurrent_download_limit { - break; - } } } ControlFlow::Continue(true) @@ -6362,6 +6359,88 @@ mod tests { test_state_with_store(store) } + #[tokio::test] + async fn download_limit_local_read_mode_waits_instead_of_proxying() { + use crate::storage::super_block::ReplicaPlacement; + + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = volume_test_state(&tmp); + { + let s = Arc::get_mut(&mut state).unwrap(); + s.read_mode = ReadMode::Local; + s.concurrent_download_limit = 1; + s.inflight_download_data_timeout = std::time::Duration::from_millis(50); + } + *state.current_master_url.write().await = "127.0.0.1:1".to_string(); + state.inflight_download_bytes.store(2, Ordering::Relaxed); + state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1 + .set_replica_placement(ReplicaPlacement::from_string("001").unwrap()) + .unwrap(); + + let path = "/1,0123456789abcdef"; + let uri: axum::http::Uri = path.parse().unwrap(); + let flow = check_download_limit( + &state, + &uri, + &HeaderMap::new(), + path, + VolumeId(1), + &Method::GET, + ) + .await; + match flow { + ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS), + ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"), + } + } + + #[tokio::test] + async fn download_limit_wait_sees_release_before_waiting() { + let mut state = streaming_test_state(); + { + let s = Arc::get_mut(&mut state).unwrap(); + s.read_mode = ReadMode::Proxy; + s.concurrent_download_limit = 1; + s.inflight_download_data_timeout = std::time::Duration::from_secs(5); + } + *state.current_master_url.write().await = "127.0.0.1:1".to_string(); + state.inflight_download_bytes.store(2, Ordering::Relaxed); + + let master_guard = state.current_master_url.write().await; + let waiter = tokio::spawn({ + let state = state.clone(); + async move { + let path = "/1,0123456789abcdef"; + let uri: axum::http::Uri = path.parse().unwrap(); + check_download_limit( + &state, + &uri, + &HeaderMap::new(), + path, + VolumeId(1), + &Method::GET, + ) + .await + } + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + state.inflight_download_bytes.store(0, Ordering::Relaxed); + state.download_notify.notify_waiters(); + drop(master_guard); + + let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter) + .await + .expect("waiter missed the released download slot") + .unwrap(); + assert!(matches!(flow, ControlFlow::Continue(true))); + } + /// Write a needle to volume 1 and return its URL path. fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String { put_test_needle_with(state, id, data, |_| {}) diff --git a/weed/command/mini.go b/weed/command/mini.go index 4b1fb6e584..4edc19673e 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -517,7 +517,7 @@ func initMiniVolumeFlags() { miniOptions.v.fileSizeLimitMB = cmdMini.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") miniOptions.v.ldbTimeout = cmdMini.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb") miniOptions.v.concurrentUploadLimitMB = cmdMini.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size") - miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size") + miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size; reads over the limit are sent to a replica unless volume.readMode is local") miniOptions.v.pprof = cmdMini.Flag.Bool("volume.pprof", false, "enable pprof http handlers") miniOptions.v.idxFolder = cmdMini.Flag.String("volume.dir.idx", "", "directory to store .idx files") miniOptions.v.inflightUploadDataTimeout = cmdMini.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout") diff --git a/weed/command/server.go b/weed/command/server.go index 78553d8ca8..04aac2f76f 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -159,7 +159,7 @@ func init() { serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.") serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited") - serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited") + serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless volume.readMode is local") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile") diff --git a/weed/command/volume.go b/weed/command/volume.go index 1388ae171d..4f51f954e2 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -240,7 +240,7 @@ func init() { v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory") v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.") v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited") - v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited") + v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless readMode is local") v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile") v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") v.metricsHttpIp = cmdVolume.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 72da689870..b5b92cf41f 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "context" "fmt" "net/http" "strconv" @@ -44,8 +45,8 @@ security settings: // Control Flow: // - No limit configured → return true (proceed normally) // - Within limit → return true (proceed normally) -// - Over limit + has replicas → proxy to replica, return false (already handled) -// - Over limit + no replicas → wait with timeout: +// - Over limit + has replicas + readMode not local → proxy to replica, return false (already handled) +// - Over limit otherwise → wait with timeout: // - Timeout → send 429 response, return false (already handled) // - Cancelled → send 499 response, return false (already handled) // - Capacity available → return true (proceed normally) @@ -73,10 +74,10 @@ func (vs *VolumeServer) checkDownloadLimit(w http.ResponseWriter, r *http.Reques // tryProxyToReplica attempts to proxy the request to a replica server if the volume has replication. // Returns: // - true: Request was handled (either proxied successfully or failed with error response) -// - false: No proxy available (volume has no replicas or request already proxied) +// - false: No proxy available (readMode is local, volume has no replicas, or request already proxied) func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request) bool { - if r.URL.Query().Get(reqIsProxied) == "true" { - return false // already proxied + if vs.ReadMode == "local" || r.URL.Query().Get(reqIsProxied) == "true" { + return false } vid, _, _, _, _ := parseURLPath(r.URL.Path) @@ -109,27 +110,30 @@ func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request // - 429 Too Many Requests: Wait timeout exceeded // - 499 Client Closed Request: Request cancelled by client func (vs *VolumeServer) waitForDownloadSlot(w http.ResponseWriter, r *http.Request) bool { - timerDownload := time.NewTimer(vs.inflightDownloadDataTimeout) - defer timerDownload.Stop() + ctx, cancel := context.WithTimeout(r.Context(), vs.inflightDownloadDataTimeout) + defer cancel() - inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize) - for inFlightDownloadSize > vs.concurrentDownloadLimit { - switch util.WaitWithTimeout(r.Context(), vs.inFlightDownloadDataLimitCond, timerDownload) { - case http.StatusTooManyRequests: - err := fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout", - r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit) - glog.V(1).Infof("too many requests: %v", err) - writeJsonError(w, r, http.StatusTooManyRequests, err) - return false - case util.HttpStatusCancelled: - glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err()) - w.WriteHeader(util.HttpStatusCancelled) - return false - } + var inFlightDownloadSize int64 + vs.inFlightDownloadDataLimitCond.L.Lock() + err := util.WaitUntil(ctx, vs.inFlightDownloadDataLimitCond, 0, func() bool { inFlightDownloadSize = atomic.LoadInt64(&vs.inFlightDownloadDataSize) stats.VolumeServerInFlightDownloadSize.Set(float64(inFlightDownloadSize)) + return inFlightDownloadSize <= vs.concurrentDownloadLimit + }) + vs.inFlightDownloadDataLimitCond.L.Unlock() + if err == nil { + return true } - return true + if r.Context().Err() != nil { + glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err()) + w.WriteHeader(util.HttpStatusCancelled) + return false + } + err = fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout", + r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit) + glog.V(1).Infof("too many requests: %v", err) + writeJsonError(w, r, http.StatusTooManyRequests, err) + return false } // checkUploadLimit handles upload concurrency limiting with timeout. diff --git a/weed/server/volume_server_handlers_download_limit_test.go b/weed/server/volume_server_handlers_download_limit_test.go new file mode 100644 index 0000000000..2319c7f3de --- /dev/null +++ b/weed/server/volume_server_handlers_download_limit_test.go @@ -0,0 +1,87 @@ +package weed_server + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/stretchr/testify/require" +) + +func TestCheckDownloadLimitLocalReadModeWaitsInsteadOfProxying(t *testing.T) { + dir := t.TempDir() + store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{dir}, []int32{10}, []util.MinFreeSpace{{}}, "", + storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig()) + done := make(chan struct{}) + go func() { + for { + select { + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.NewEcShardsChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "001", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0)) + + vs := &VolumeServer{ + store: store, + ReadMode: "local", + concurrentDownloadLimit: 1, + inFlightDownloadDataSize: 2, + inflightDownloadDataTimeout: 50 * time.Millisecond, + inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil).WithContext(ctx) + w := httptest.NewRecorder() + + require.False(t, vs.checkDownloadLimit(w, r)) + require.Equal(t, http.StatusTooManyRequests, w.Code) +} + +func TestWaitForDownloadSlotSeesReleaseBeforeWaiting(t *testing.T) { + vs := &VolumeServer{ + concurrentDownloadLimit: 1, + inFlightDownloadDataSize: 2, + inflightDownloadDataTimeout: 5 * time.Second, + inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), + } + r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil) + w := httptest.NewRecorder() + + vs.inFlightDownloadDataLimitCond.L.Lock() + got := make(chan bool, 1) + go func() { got <- vs.waitForDownloadSlot(w, r) }() + time.Sleep(100 * time.Millisecond) + atomic.AddInt64(&vs.inFlightDownloadDataSize, -2) + vs.inFlightDownloadDataLimitCond.Broadcast() + vs.inFlightDownloadDataLimitCond.L.Unlock() + + select { + case ok := <-got: + require.True(t, ok) + case <-time.After(2 * time.Second): + t.Fatal("waiter missed the released download slot") + } +} diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 30d6968b52..ceb2f52639 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -266,7 +266,9 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) defer func() { atomic.AddInt64(&vs.inFlightDownloadDataSize, -int64(memoryCost)) if vs.concurrentDownloadLimit != 0 { + vs.inFlightDownloadDataLimitCond.L.Lock() vs.inFlightDownloadDataLimitCond.Broadcast() + vs.inFlightDownloadDataLimitCond.L.Unlock() } }()