volume: keep reads over the download limit local when readMode is local (#12679)

* volume: keep reads over the download limit local when readMode is local

With -readMode=local the server never sends a read for a volume it does
not hold elsewhere, but a read over -concurrentDownloadLimitMB on a
replicated volume was still redirected to another replica. Skip the
replica fallback in local mode, so the read waits for a download slot
and answers 429 when the wait times out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): keep reads over the download limit local when readMode is local

Mirror the Go volume server: with readMode local, a read over the
download limit waits for a slot instead of being redirected to a
replica. proxy_or_redirect_to_target no longer needs the local-mode
redirect switch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume: check download capacity under the wait lock

waitForDownloadSlot read the in-flight size outside the cond lock and
the read handler broadcast without it, so a download finishing between
the check and the wait was missed and the read waited out the timeout
for a 429. Check capacity with util.WaitUntil under the lock and
broadcast under it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): register for the download notification before checking capacity

The wait loop created its Notified future only after the capacity
check, so a body dropped in between had its notify_waiters() lost and
the read waited out the deadline for a 429. Create the future first,
then check, as Go now checks under the cond lock.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-16 16:20:15 -08:00
1 parent f4e638fa1d
commit 9e5357782e
8 files changed
+215 -42

No files matched your search

+2 -1
View File
@@ -138,7 +138,8 @@ pub struct Cli {
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
pub concurrent_upload_limit_mb: u32,
/// Limit total concurrent download size in MB, 0 means unlimited.
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
/// the limit are sent to a replica unless readMode is local.
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
pub concurrent_download_limit_mb: u32,
+95 -16
View File
@@ -913,7 +913,6 @@ async fn proxy_or_redirect_to_target(
state: &VolumeServerState,
info: ProxyRequestInfo,
vid: VolumeId,
allow_local_redirect: bool,
) -> Response {
// Look up volume locations from master
let locations = match lookup_volume(
@@ -961,10 +960,7 @@ async fn proxy_or_redirect_to_target(
match state.read_mode {
ReadMode::Proxy => proxy_request(state, &info, target).await,
ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme),
ReadMode::Local if allow_local_redirect => {
redirect_request(&info, target, &state.outgoing_http_scheme)
}
ReadMode::Local => unreachable!(),
ReadMode::Local => StatusCode::NOT_FOUND.into_response(),
}
}
@@ -1431,7 +1427,7 @@ async fn proxy_missing_volume(
None => return StatusCode::NOT_FOUND.into_response(),
};
proxy_or_redirect_to_target(state, info, vid, false).await
proxy_or_redirect_to_target(state, info, vid).await
}
/// Go's `proxied` query param check (loop prevention).
@@ -1464,8 +1460,10 @@ async fn check_download_limit(
// Go tries proxy to replica once before the wait loop
// (checkDownloadLimit); it does not retry on each wakeup.
let should_try_replica =
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
let should_try_replica = state.read_mode != ReadMode::Local
&& !is_proxied_query(&query_string)
&& !state.current_master().await.is_empty()
&& {
let store = state.store.read().unwrap();
store
.find_volume(vid)
@@ -1475,25 +1473,24 @@ async fn check_download_limit(
&& let Some(info) =
build_proxy_request_info(path, request_headers, &query_string, method)
{
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await);
}
// Blocking wait loop (Go's waitForDownloadSlot)
loop {
if tokio::time::timeout_at(deadline, state.download_notify.notified())
.await
.is_err()
let notified = state.download_notify.notified();
if state.inflight_download_bytes.load(Ordering::Relaxed)
<= state.concurrent_download_limit
{
break;
}
if tokio::time::timeout_at(deadline, notified).await.is_err() {
return ControlFlow::Break(json_error_with_query(
StatusCode::TOO_MANY_REQUESTS,
"download limit exceeded",
uri.query(),
));
}
let current = state.inflight_download_bytes.load(Ordering::Relaxed);
if current <= state.concurrent_download_limit {
break;
}
}
}
ControlFlow::Continue(true)
@@ -6362,6 +6359,88 @@ mod tests {
test_state_with_store(store)
}
#[tokio::test]
async fn download_limit_local_read_mode_waits_instead_of_proxying() {
use crate::storage::super_block::ReplicaPlacement;
let tmp = tempfile::TempDir::new().unwrap();
let mut state = volume_test_state(&tmp);
{
let s = Arc::get_mut(&mut state).unwrap();
s.read_mode = ReadMode::Local;
s.concurrent_download_limit = 1;
s.inflight_download_data_timeout = std::time::Duration::from_millis(50);
}
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
state.inflight_download_bytes.store(2, Ordering::Relaxed);
state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
.unwrap();
let path = "/1,0123456789abcdef";
let uri: axum::http::Uri = path.parse().unwrap();
let flow = check_download_limit(
&state,
&uri,
&HeaderMap::new(),
path,
VolumeId(1),
&Method::GET,
)
.await;
match flow {
ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS),
ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"),
}
}
#[tokio::test]
async fn download_limit_wait_sees_release_before_waiting() {
let mut state = streaming_test_state();
{
let s = Arc::get_mut(&mut state).unwrap();
s.read_mode = ReadMode::Proxy;
s.concurrent_download_limit = 1;
s.inflight_download_data_timeout = std::time::Duration::from_secs(5);
}
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
state.inflight_download_bytes.store(2, Ordering::Relaxed);
let master_guard = state.current_master_url.write().await;
let waiter = tokio::spawn({
let state = state.clone();
async move {
let path = "/1,0123456789abcdef";
let uri: axum::http::Uri = path.parse().unwrap();
check_download_limit(
&state,
&uri,
&HeaderMap::new(),
path,
VolumeId(1),
&Method::GET,
)
.await
}
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
state.inflight_download_bytes.store(0, Ordering::Relaxed);
state.download_notify.notify_waiters();
drop(master_guard);
let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
.await
.expect("waiter missed the released download slot")
.unwrap();
assert!(matches!(flow, ControlFlow::Continue(true)));
}
/// Write a needle to volume 1 and return its URL path.
fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
put_test_needle_with(state, id, data, |_| {})
+1 -1
View File
@@ -517,7 +517,7 @@ func initMiniVolumeFlags() {
miniOptions.v.fileSizeLimitMB = cmdMini.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
miniOptions.v.ldbTimeout = cmdMini.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb")
miniOptions.v.concurrentUploadLimitMB = cmdMini.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size")
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size")
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size; reads over the limit are sent to a replica unless volume.readMode is local")
miniOptions.v.pprof = cmdMini.Flag.Bool("volume.pprof", false, "enable pprof http handlers")
miniOptions.v.idxFolder = cmdMini.Flag.String("volume.dir.idx", "", "directory to store .idx files")
miniOptions.v.inflightUploadDataTimeout = cmdMini.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout")
+1 -1
View File
@@ -159,7 +159,7 @@ func init() {
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless volume.readMode is local")
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
+1 -1
View File
@@ -240,7 +240,7 @@ func init() {
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless readMode is local")
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.metricsHttpIp = cmdVolume.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
+26 -22
View File
@@ -1,6 +1,7 @@
package weed_server
import (
"context"
"fmt"
"net/http"
"strconv"
@@ -44,8 +45,8 @@ security settings:
// Control Flow:
// - No limit configured → return true (proceed normally)
// - Within limit → return true (proceed normally)
// - Over limit + has replicas → proxy to replica, return false (already handled)
// - Over limit + no replicas → wait with timeout:
// - Over limit + has replicas + readMode not local → proxy to replica, return false (already handled)
// - Over limit otherwise → wait with timeout:
// - Timeout → send 429 response, return false (already handled)
// - Cancelled → send 499 response, return false (already handled)
// - Capacity available → return true (proceed normally)
@@ -73,10 +74,10 @@ func (vs *VolumeServer) checkDownloadLimit(w http.ResponseWriter, r *http.Reques
// tryProxyToReplica attempts to proxy the request to a replica server if the volume has replication.
// Returns:
// - true: Request was handled (either proxied successfully or failed with error response)
// - false: No proxy available (volume has no replicas or request already proxied)
// - false: No proxy available (readMode is local, volume has no replicas, or request already proxied)
func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request) bool {
if r.URL.Query().Get(reqIsProxied) == "true" {
return false // already proxied
if vs.ReadMode == "local" || r.URL.Query().Get(reqIsProxied) == "true" {
return false
}
vid, _, _, _, _ := parseURLPath(r.URL.Path)
@@ -109,27 +110,30 @@ func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request
// - 429 Too Many Requests: Wait timeout exceeded
// - 499 Client Closed Request: Request cancelled by client
func (vs *VolumeServer) waitForDownloadSlot(w http.ResponseWriter, r *http.Request) bool {
timerDownload := time.NewTimer(vs.inflightDownloadDataTimeout)
defer timerDownload.Stop()
ctx, cancel := context.WithTimeout(r.Context(), vs.inflightDownloadDataTimeout)
defer cancel()
inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
for inFlightDownloadSize > vs.concurrentDownloadLimit {
switch util.WaitWithTimeout(r.Context(), vs.inFlightDownloadDataLimitCond, timerDownload) {
case http.StatusTooManyRequests:
err := fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout",
r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit)
glog.V(1).Infof("too many requests: %v", err)
writeJsonError(w, r, http.StatusTooManyRequests, err)
return false
case util.HttpStatusCancelled:
glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err())
w.WriteHeader(util.HttpStatusCancelled)
return false
}
var inFlightDownloadSize int64
vs.inFlightDownloadDataLimitCond.L.Lock()
err := util.WaitUntil(ctx, vs.inFlightDownloadDataLimitCond, 0, func() bool {
inFlightDownloadSize = atomic.LoadInt64(&vs.inFlightDownloadDataSize)
stats.VolumeServerInFlightDownloadSize.Set(float64(inFlightDownloadSize))
return inFlightDownloadSize <= vs.concurrentDownloadLimit
})
vs.inFlightDownloadDataLimitCond.L.Unlock()
if err == nil {
return true
}
return true
if r.Context().Err() != nil {
glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err())
w.WriteHeader(util.HttpStatusCancelled)
return false
}
err = fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout",
r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit)
glog.V(1).Infof("too many requests: %v", err)
writeJsonError(w, r, http.StatusTooManyRequests, err)
return false
}
// checkUploadLimit handles upload concurrency limiting with timeout.
@@ -0,0 +1,87 @@
package weed_server
import (
"context"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/require"
)
func TestCheckDownloadLimitLocalReadModeWaitsInsteadOfProxying(t *testing.T) {
dir := t.TempDir()
store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
[]string{dir}, []int32{10}, []util.MinFreeSpace{{}}, "",
storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
done := make(chan struct{})
go func() {
for {
select {
case <-store.NewVolumesChan:
case <-store.DeletedVolumesChan:
case <-store.NewEcShardsChan:
case <-store.DeletedEcShardsChan:
case <-store.StateUpdateChan:
case <-done:
return
}
}
}()
t.Cleanup(func() {
store.Close()
close(done)
})
require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "001", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
vs := &VolumeServer{
store: store,
ReadMode: "local",
concurrentDownloadLimit: 1,
inFlightDownloadDataSize: 2,
inflightDownloadDataTimeout: 50 * time.Millisecond,
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil).WithContext(ctx)
w := httptest.NewRecorder()
require.False(t, vs.checkDownloadLimit(w, r))
require.Equal(t, http.StatusTooManyRequests, w.Code)
}
func TestWaitForDownloadSlotSeesReleaseBeforeWaiting(t *testing.T) {
vs := &VolumeServer{
concurrentDownloadLimit: 1,
inFlightDownloadDataSize: 2,
inflightDownloadDataTimeout: 5 * time.Second,
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
}
r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil)
w := httptest.NewRecorder()
vs.inFlightDownloadDataLimitCond.L.Lock()
got := make(chan bool, 1)
go func() { got <- vs.waitForDownloadSlot(w, r) }()
time.Sleep(100 * time.Millisecond)
atomic.AddInt64(&vs.inFlightDownloadDataSize, -2)
vs.inFlightDownloadDataLimitCond.Broadcast()
vs.inFlightDownloadDataLimitCond.L.Unlock()
select {
case ok := <-got:
require.True(t, ok)
case <-time.After(2 * time.Second):
t.Fatal("waiter missed the released download slot")
}
}
@@ -266,7 +266,9 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
defer func() {
atomic.AddInt64(&vs.inFlightDownloadDataSize, -int64(memoryCost))
if vs.concurrentDownloadLimit != 0 {
vs.inFlightDownloadDataLimitCond.L.Lock()
vs.inFlightDownloadDataLimitCond.Broadcast()
vs.inFlightDownloadDataLimitCond.L.Unlock()
}
}()