mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 01:37:10 +01:00
volume: keep reads over the download limit local when readMode is local (#12679)
* volume: keep reads over the download limit local when readMode is local With -readMode=local the server never sends a read for a volume it does not hold elsewhere, but a read over -concurrentDownloadLimitMB on a replicated volume was still redirected to another replica. Skip the replica fallback in local mode, so the read waits for a download slot and answers 429 when the wait times out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): keep reads over the download limit local when readMode is local Mirror the Go volume server: with readMode local, a read over the download limit waits for a slot instead of being redirected to a replica. proxy_or_redirect_to_target no longer needs the local-mode redirect switch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume: check download capacity under the wait lock waitForDownloadSlot read the in-flight size outside the cond lock and the read handler broadcast without it, so a download finishing between the check and the wait was missed and the read waited out the timeout for a 429. Check capacity with util.WaitUntil under the lock and broadcast under it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): register for the download notification before checking capacity The wait loop created its Notified future only after the capacity check, so a body dropped in between had its notify_waiters() lost and the read waited out the deadline for a 429. Create the future first, then check, as Go now checks under the cond lock. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f4e638fa1d
commit
9e5357782e
8 files changed
+215
-42
No files matched your search
@@ -138,7 +138,8 @@ pub struct Cli {
|
||||
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_upload_limit_mb: u32,
|
||||
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited.
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
|
||||
/// the limit are sent to a replica unless readMode is local.
|
||||
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_download_limit_mb: u32,
|
||||
|
||||
|
||||
@@ -913,7 +913,6 @@ async fn proxy_or_redirect_to_target(
|
||||
state: &VolumeServerState,
|
||||
info: ProxyRequestInfo,
|
||||
vid: VolumeId,
|
||||
allow_local_redirect: bool,
|
||||
) -> Response {
|
||||
// Look up volume locations from master
|
||||
let locations = match lookup_volume(
|
||||
@@ -961,10 +960,7 @@ async fn proxy_or_redirect_to_target(
|
||||
match state.read_mode {
|
||||
ReadMode::Proxy => proxy_request(state, &info, target).await,
|
||||
ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme),
|
||||
ReadMode::Local if allow_local_redirect => {
|
||||
redirect_request(&info, target, &state.outgoing_http_scheme)
|
||||
}
|
||||
ReadMode::Local => unreachable!(),
|
||||
ReadMode::Local => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1431,7 +1427,7 @@ async fn proxy_missing_volume(
|
||||
None => return StatusCode::NOT_FOUND.into_response(),
|
||||
};
|
||||
|
||||
proxy_or_redirect_to_target(state, info, vid, false).await
|
||||
proxy_or_redirect_to_target(state, info, vid).await
|
||||
}
|
||||
|
||||
/// Go's `proxied` query param check (loop prevention).
|
||||
@@ -1464,8 +1460,10 @@ async fn check_download_limit(
|
||||
|
||||
// Go tries proxy to replica once before the wait loop
|
||||
// (checkDownloadLimit); it does not retry on each wakeup.
|
||||
let should_try_replica =
|
||||
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
|
||||
let should_try_replica = state.read_mode != ReadMode::Local
|
||||
&& !is_proxied_query(&query_string)
|
||||
&& !state.current_master().await.is_empty()
|
||||
&& {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
@@ -1475,25 +1473,24 @@ async fn check_download_limit(
|
||||
&& let Some(info) =
|
||||
build_proxy_request_info(path, request_headers, &query_string, method)
|
||||
{
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await);
|
||||
}
|
||||
|
||||
// Blocking wait loop (Go's waitForDownloadSlot)
|
||||
loop {
|
||||
if tokio::time::timeout_at(deadline, state.download_notify.notified())
|
||||
.await
|
||||
.is_err()
|
||||
let notified = state.download_notify.notified();
|
||||
if state.inflight_download_bytes.load(Ordering::Relaxed)
|
||||
<= state.concurrent_download_limit
|
||||
{
|
||||
break;
|
||||
}
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
return ControlFlow::Break(json_error_with_query(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"download limit exceeded",
|
||||
uri.query(),
|
||||
));
|
||||
}
|
||||
let current = state.inflight_download_bytes.load(Ordering::Relaxed);
|
||||
if current <= state.concurrent_download_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ControlFlow::Continue(true)
|
||||
@@ -6362,6 +6359,88 @@ mod tests {
|
||||
test_state_with_store(store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_local_read_mode_waits_instead_of_proxying() {
|
||||
use crate::storage::super_block::ReplicaPlacement;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mut state = volume_test_state(&tmp);
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Local;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_millis(50);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
|
||||
.unwrap();
|
||||
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
let flow = check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await;
|
||||
match flow {
|
||||
ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS),
|
||||
ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_wait_sees_release_before_waiting() {
|
||||
let mut state = streaming_test_state();
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Proxy;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_secs(5);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
|
||||
let master_guard = state.current_master_url.write().await;
|
||||
let waiter = tokio::spawn({
|
||||
let state = state.clone();
|
||||
async move {
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
state.inflight_download_bytes.store(0, Ordering::Relaxed);
|
||||
state.download_notify.notify_waiters();
|
||||
drop(master_guard);
|
||||
|
||||
let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
|
||||
.await
|
||||
.expect("waiter missed the released download slot")
|
||||
.unwrap();
|
||||
assert!(matches!(flow, ControlFlow::Continue(true)));
|
||||
}
|
||||
|
||||
/// Write a needle to volume 1 and return its URL path.
|
||||
fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
|
||||
put_test_needle_with(state, id, data, |_| {})
|
||||
|
||||
@@ -517,7 +517,7 @@ func initMiniVolumeFlags() {
|
||||
miniOptions.v.fileSizeLimitMB = cmdMini.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
miniOptions.v.ldbTimeout = cmdMini.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb")
|
||||
miniOptions.v.concurrentUploadLimitMB = cmdMini.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size")
|
||||
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size")
|
||||
miniOptions.v.concurrentDownloadLimitMB = cmdMini.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size; reads over the limit are sent to a replica unless volume.readMode is local")
|
||||
miniOptions.v.pprof = cmdMini.Flag.Bool("volume.pprof", false, "enable pprof http handlers")
|
||||
miniOptions.v.idxFolder = cmdMini.Flag.String("volume.dir.idx", "", "directory to store .idx files")
|
||||
miniOptions.v.inflightUploadDataTimeout = cmdMini.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout")
|
||||
|
||||
@@ -159,7 +159,7 @@ func init() {
|
||||
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
serverOptions.v.ldbTimeout = cmdServer.Flag.Int64("volume.index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
||||
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
|
||||
serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless volume.readMode is local")
|
||||
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
|
||||
serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
|
||||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
|
||||
|
||||
@@ -240,7 +240,7 @@ func init() {
|
||||
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
|
||||
v.ldbTimeout = cmdVolume.Flag.Int64("index.leveldbTimeout", 0, "alive time for leveldb (default to 0). If leveldb of volume is not accessed in ldbTimeout hours, it will be off loaded to reduce opened files and memory consumption.")
|
||||
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
||||
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited")
|
||||
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 0, "limit total concurrent download size, 0 means unlimited; reads over the limit are sent to a replica unless readMode is local")
|
||||
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes -memprofile and -cpuprofile")
|
||||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.metricsHttpIp = cmdVolume.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -44,8 +45,8 @@ security settings:
|
||||
// Control Flow:
|
||||
// - No limit configured → return true (proceed normally)
|
||||
// - Within limit → return true (proceed normally)
|
||||
// - Over limit + has replicas → proxy to replica, return false (already handled)
|
||||
// - Over limit + no replicas → wait with timeout:
|
||||
// - Over limit + has replicas + readMode not local → proxy to replica, return false (already handled)
|
||||
// - Over limit otherwise → wait with timeout:
|
||||
// - Timeout → send 429 response, return false (already handled)
|
||||
// - Cancelled → send 499 response, return false (already handled)
|
||||
// - Capacity available → return true (proceed normally)
|
||||
@@ -73,10 +74,10 @@ func (vs *VolumeServer) checkDownloadLimit(w http.ResponseWriter, r *http.Reques
|
||||
// tryProxyToReplica attempts to proxy the request to a replica server if the volume has replication.
|
||||
// Returns:
|
||||
// - true: Request was handled (either proxied successfully or failed with error response)
|
||||
// - false: No proxy available (volume has no replicas or request already proxied)
|
||||
// - false: No proxy available (readMode is local, volume has no replicas, or request already proxied)
|
||||
func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request) bool {
|
||||
if r.URL.Query().Get(reqIsProxied) == "true" {
|
||||
return false // already proxied
|
||||
if vs.ReadMode == "local" || r.URL.Query().Get(reqIsProxied) == "true" {
|
||||
return false
|
||||
}
|
||||
|
||||
vid, _, _, _, _ := parseURLPath(r.URL.Path)
|
||||
@@ -109,27 +110,30 @@ func (vs *VolumeServer) tryProxyToReplica(w http.ResponseWriter, r *http.Request
|
||||
// - 429 Too Many Requests: Wait timeout exceeded
|
||||
// - 499 Client Closed Request: Request cancelled by client
|
||||
func (vs *VolumeServer) waitForDownloadSlot(w http.ResponseWriter, r *http.Request) bool {
|
||||
timerDownload := time.NewTimer(vs.inflightDownloadDataTimeout)
|
||||
defer timerDownload.Stop()
|
||||
ctx, cancel := context.WithTimeout(r.Context(), vs.inflightDownloadDataTimeout)
|
||||
defer cancel()
|
||||
|
||||
inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
|
||||
for inFlightDownloadSize > vs.concurrentDownloadLimit {
|
||||
switch util.WaitWithTimeout(r.Context(), vs.inFlightDownloadDataLimitCond, timerDownload) {
|
||||
case http.StatusTooManyRequests:
|
||||
err := fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout",
|
||||
r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit)
|
||||
glog.V(1).Infof("too many requests: %v", err)
|
||||
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
||||
return false
|
||||
case util.HttpStatusCancelled:
|
||||
glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err())
|
||||
w.WriteHeader(util.HttpStatusCancelled)
|
||||
return false
|
||||
}
|
||||
var inFlightDownloadSize int64
|
||||
vs.inFlightDownloadDataLimitCond.L.Lock()
|
||||
err := util.WaitUntil(ctx, vs.inFlightDownloadDataLimitCond, 0, func() bool {
|
||||
inFlightDownloadSize = atomic.LoadInt64(&vs.inFlightDownloadDataSize)
|
||||
stats.VolumeServerInFlightDownloadSize.Set(float64(inFlightDownloadSize))
|
||||
return inFlightDownloadSize <= vs.concurrentDownloadLimit
|
||||
})
|
||||
vs.inFlightDownloadDataLimitCond.L.Unlock()
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
return true
|
||||
if r.Context().Err() != nil {
|
||||
glog.V(1).Infof("request %s cancelled from %s: %v", r.URL.Path, r.RemoteAddr, r.Context().Err())
|
||||
w.WriteHeader(util.HttpStatusCancelled)
|
||||
return false
|
||||
}
|
||||
err = fmt.Errorf("request %s because inflight download data %d > %d, and wait timeout",
|
||||
r.URL.Path, inFlightDownloadSize, vs.concurrentDownloadLimit)
|
||||
glog.V(1).Infof("too many requests: %v", err)
|
||||
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
||||
return false
|
||||
}
|
||||
|
||||
// checkUploadLimit handles upload concurrency limiting with timeout.
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCheckDownloadLimitLocalReadModeWaitsInsteadOfProxying(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
|
||||
[]string{dir}, []int32{10}, []util.MinFreeSpace{{}}, "",
|
||||
storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-store.NewVolumesChan:
|
||||
case <-store.DeletedVolumesChan:
|
||||
case <-store.NewEcShardsChan:
|
||||
case <-store.DeletedEcShardsChan:
|
||||
case <-store.StateUpdateChan:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
store.Close()
|
||||
close(done)
|
||||
})
|
||||
require.NoError(t, store.AddVolume(1, "", storage.NeedleMapInMemory, "001", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
||||
|
||||
vs := &VolumeServer{
|
||||
store: store,
|
||||
ReadMode: "local",
|
||||
concurrentDownloadLimit: 1,
|
||||
inFlightDownloadDataSize: 2,
|
||||
inflightDownloadDataTimeout: 50 * time.Millisecond,
|
||||
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil).WithContext(ctx)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
require.False(t, vs.checkDownloadLimit(w, r))
|
||||
require.Equal(t, http.StatusTooManyRequests, w.Code)
|
||||
}
|
||||
|
||||
func TestWaitForDownloadSlotSeesReleaseBeforeWaiting(t *testing.T) {
|
||||
vs := &VolumeServer{
|
||||
concurrentDownloadLimit: 1,
|
||||
inFlightDownloadDataSize: 2,
|
||||
inflightDownloadDataTimeout: 5 * time.Second,
|
||||
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodGet, "/1,0123456789abcdef", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
vs.inFlightDownloadDataLimitCond.L.Lock()
|
||||
got := make(chan bool, 1)
|
||||
go func() { got <- vs.waitForDownloadSlot(w, r) }()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
atomic.AddInt64(&vs.inFlightDownloadDataSize, -2)
|
||||
vs.inFlightDownloadDataLimitCond.Broadcast()
|
||||
vs.inFlightDownloadDataLimitCond.L.Unlock()
|
||||
|
||||
select {
|
||||
case ok := <-got:
|
||||
require.True(t, ok)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("waiter missed the released download slot")
|
||||
}
|
||||
}
|
||||
@@ -266,7 +266,9 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
||||
defer func() {
|
||||
atomic.AddInt64(&vs.inFlightDownloadDataSize, -int64(memoryCost))
|
||||
if vs.concurrentDownloadLimit != 0 {
|
||||
vs.inFlightDownloadDataLimitCond.L.Lock()
|
||||
vs.inFlightDownloadDataLimitCond.Broadcast()
|
||||
vs.inFlightDownloadDataLimitCond.L.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
Reference in new issue
Block a user