Compare commits

...
Author SHA1 Message Date
user.emailandDevin 6389887ebb seaweed-volume: drop proxy-specific headers from replica proxy reads
Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-26 02:56:46 -07:00
user.email 51d068a98d volume: drop proxy-specific headers from replica proxy reads 2026-10-26 02:56:20 -07:00
user.email 29d00ae794 volume: skip the proxy lookup error when the inbound request is gone 2026-10-26 02:55:56 -07:00
user.email 21c4de24e9 seaweed-volume: drop Connection-nominated headers from proxy reads 2026-10-26 02:44:02 -07:00
user.email f54f20e654 volume: drop hop-by-hop and Connection-nominated headers from proxy reads 2026-10-26 02:44:02 -07:00
user.email 561307a7ca volume: bound the proxy master lookup to the request context 2026-10-26 02:44:02 -07:00
chrislusf fdd9c1edd3 seaweed-volume: drop hop-by-hop headers from replica proxy reads
Go's inbound request drops Host, Content-Length and Transfer-Encoding
into dedicated fields, so its forwarded header set never carries them.
hyper keeps them; forwarding a client's Content-Length with no body
stalls the replica's read, and forwarding its Host misnames the target.
2026-10-26 02:01:05 -07:00
chrislusf d09c4d951a seaweed-volume: answered lookup errors are missing data, not failures
A master reply carrying a per-volume error with HTTP 404 (volume not
found) now yields empty locations like Go's LookupResult, so proxy mode
answers 404. Other failure statuses such as 503 during master warmup
remain lookup failures that proxy mode answers 500.
2026-10-26 01:58:48 -07:00
chrislusf f03806064e volume: guard nil lookup result in proxy path
A master response that omits the requested vid leaves
LookupVolumeId returning a nil *LookupResult with a nil error;
dereferencing it panics the read handler.
2026-10-26 01:58:48 -07:00
user.email 666f80564f seaweed-volume: answer 404 for missing volumes in proxy-mode lookups
Mirror Go's LookupVolumeId: a per-volume error in the master's lookup
reply (e.g. a missing volume) means "no locations" and answers 404
downstream, not a lookup failure that proxy mode answers 500.
2026-10-26 01:57:16 -07:00
chrislusf 2b47004269 volume: bound the proxied-request wait in the method-preservation test 2026-10-26 01:49:06 -07:00
chrislusf 1ef76dfb12 volume: keep master lookup error detail out of the proxy error response 2026-10-26 01:49:06 -07:00
chrislusf 18b71ed0ef seaweed-volume: forward inbound method and surface lookup failure in proxy reads
Mirror the Go handler: proxy_request used to always send GET, turning a client HEAD into a replica download, and a failed master lookup returned 404 like missing data. Forward the inbound method and answer 500 for lookup failures in proxy mode.
2026-10-26 01:47:04 -07:00
chrislusf a7b0aa8926 volume: shuffle a copy of cached replica locations for proxy reads
LookupVolumeId serves cached location slices shared between lookups; shuffling in place races with concurrent proxied reads of the same volume.
2026-10-26 01:47:00 -07:00
shr 63dfda3412 fix 12074 2026-10-26 16:08:14 +08:00
4 changed files with 672 additions and 23 deletions

No files matched your search

+316 -15
View File
@@ -555,6 +555,7 @@ async fn lookup_volume(
.send()
.await
.map_err(|e| format!("lookup request failed: {}", e))?;
let status = resp.status();
let result: LookupResult = resp
.json()
.await
@@ -562,7 +563,14 @@ async fn lookup_volume(
if let Some(err) = result.error
&& !err.is_empty()
{
return Err(err);
// Mirror Go: a missing volume arrives as a per-volume error in an
// answered lookup (HTTP 404), meaning "no locations" — a real master
// failure (e.g. 503 while warming up) stays an error so proxy mode
// can tell missing data (404) from a failed lookup (500).
if status != StatusCode::NOT_FOUND {
return Err(err);
}
tracing::debug!("volume {} lookup: {}", volume_id, err);
}
Ok(result.locations.unwrap_or_default())
}
@@ -812,6 +820,7 @@ async fn do_replicated_request(
/// Extracted request info needed for proxy/redirect (avoids borrowing Request across await).
struct ProxyRequestInfo {
method: Method,
original_headers: HeaderMap,
original_query: String,
path: String,
@@ -823,12 +832,14 @@ fn build_proxy_request_info(
path: &str,
headers: &HeaderMap,
query_string: &str,
method: &Method,
) -> Option<ProxyRequestInfo> {
// Go redirects to "vid,fid" built from parseURLPath, so the extension must
// already be off the fid here (volume_server_handlers_read.go:128-137).
let parsed = parse_needle_path(path)?;
Some(ProxyRequestInfo {
method: method.clone(),
original_headers: headers.clone(),
original_query: query_string.to_string(),
path: path.to_string(),
@@ -856,7 +867,11 @@ async fn proxy_or_redirect_to_target(
Ok(locs) => locs,
Err(e) => {
tracing::warn!("volume lookup failed for {}: {}", vid.0, e);
return StatusCode::NOT_FOUND.into_response();
// Go: master lookup failure is 500 in proxy mode, 404 for redirect
return match state.read_mode {
ReadMode::Proxy => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
_ => StatusCode::NOT_FOUND.into_response(),
};
}
};
@@ -920,11 +935,43 @@ async fn proxy_request(
}
};
// Build the proxy request
let mut req_builder = state.http_client.get(&target_url);
// Build the proxy request, preserving the inbound method (Go parity:
// a client HEAD must not turn into a GET download at the replica).
let mut req_builder = state.http_client.request(info.method.clone(), &target_url);
// Forward all original headers as raw bytes, as Go does.
// Forward the original headers, as Go does — minus hop-by-hop fields.
// Host must be the replica's, and Content-Length/Transfer-Encoding describe
// a body this request does not carry; forwarding them stalls the peer's
// read. Connection also nominates per-request fields to drop (RFC 9110
// 7.6.1).
let mut hop_by_hop: std::collections::HashSet<String> = info
.original_headers
.get_all(header::CONNECTION)
.iter()
.filter_map(|v| v.to_str().ok())
.flat_map(|v| v.split(','))
.map(|token| token.trim().to_ascii_lowercase())
.filter(|token| !token.is_empty())
.collect();
for fixed in [
"host",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
"te",
"trailer",
"upgrade",
"proxy-connection",
"proxy-authenticate",
"proxy-authorization",
] {
hop_by_hop.insert(fixed.to_string());
}
for (name, value) in &info.original_headers {
if hop_by_hop.contains(name.as_str()) {
continue;
}
req_builder = req_builder.header(name.clone(), value.clone());
}
@@ -1088,14 +1135,30 @@ async fn get_or_head_handler_inner(
let has_ec_volume = state.store.read().unwrap().has_ec_volume(vid);
if !has_volume && !has_ec_volume {
return proxy_missing_volume(&state, request.uri(), request.headers(), &path, vid).await;
return proxy_missing_volume(
&state,
request.uri(),
request.headers(),
&path,
vid,
&method,
)
.await;
}
let track_download =
match check_download_limit(&state, request.uri(), request.headers(), &path, vid).await {
ControlFlow::Continue(track_download) => track_download,
ControlFlow::Break(resp) => return resp,
};
let track_download = match check_download_limit(
&state,
request.uri(),
request.headers(),
&path,
vid,
&method,
)
.await
{
ControlFlow::Continue(track_download) => track_download,
ControlFlow::Break(resp) => return resp,
};
let read_deleted = query.read_deleted.as_deref() == Some("true");
// Go reads Range once and ignores an empty value; a non-UTF-8 byte fails its parse.
@@ -1277,6 +1340,7 @@ async fn proxy_missing_volume(
request_headers: &HeaderMap,
path: &str,
vid: VolumeId,
method: &Method,
) -> Response {
let query_string = uri.query().unwrap_or("").to_string();
if is_proxied_query(&query_string)
@@ -1287,7 +1351,7 @@ async fn proxy_missing_volume(
}
// For redirect, fid must be stripped of extension (Go parity: parseURLPath returns raw fid).
let info = match build_proxy_request_info(path, request_headers, &query_string) {
let info = match build_proxy_request_info(path, request_headers, &query_string, method) {
Some(info) => info,
None => return StatusCode::NOT_FOUND.into_response(),
};
@@ -1308,6 +1372,7 @@ async fn check_download_limit(
request_headers: &HeaderMap,
path: &str,
vid: VolumeId,
method: &Method,
) -> ControlFlow<Response, bool> {
if state.concurrent_download_limit <= 0 {
return ControlFlow::Continue(false);
@@ -1332,7 +1397,8 @@ async fn check_download_limit(
.is_some_and(|(_, vol)| vol.super_block().replica_placement.has_replication())
};
if should_try_replica
&& let Some(info) = build_proxy_request_info(path, request_headers, &query_string)
&& let Some(info) =
build_proxy_request_info(path, request_headers, &query_string, method)
{
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
}
@@ -5181,6 +5247,7 @@ mod tests {
#[test]
fn test_redirect_request_uses_outgoing_http_scheme() {
let info = ProxyRequestInfo {
method: Method::GET,
original_headers: HeaderMap::new(),
original_query: "collection=photos&readDeleted=true".to_string(),
path: "/3,01637037d6".to_string(),
@@ -5209,6 +5276,7 @@ mod tests {
#[test]
fn test_redirect_request_strips_grpc_port_from_target_url() {
let info = ProxyRequestInfo {
method: Method::GET,
original_headers: HeaderMap::new(),
original_query: String::new(),
path: "/3,01637037d6".to_string(),
@@ -5240,7 +5308,8 @@ mod tests {
#[test]
fn test_redirect_location_drops_extension_for_slash_form() {
let info =
build_proxy_request_info("/3/01637037d6.jpg", &HeaderMap::new(), "").expect("parses");
build_proxy_request_info("/3/01637037d6.jpg", &HeaderMap::new(), "", &Method::GET)
.expect("parses");
assert_eq!(info.vid_str, "3");
assert_eq!(info.fid_str, "01637037d6");
@@ -5334,6 +5403,60 @@ mod tests {
server.abort();
}
/// The master answers a missing volume with 404 and a per-volume `error`
/// in the lookup body. Go's LookupResult.Error never becomes the Go error
/// return — the volume server treats it as empty locations and replies
/// 404. Surfacing it as Err would make proxy-mode reads answer 500.
/// A lookup answered with any other failure status (e.g. 503 while the
/// master warms up) stays an error, like Go's transport-level failure.
#[tokio::test]
async fn test_lookup_volume_missing_volume_returns_empty() {
use axum::{Router, routing::get};
let app = Router::new().route(
"/dir/lookup",
get(|axum::extract::Query(params): axum::extract::Query<
std::collections::HashMap<String, String>,
>| async move {
let (status, error) = match params.get("volumeId").map(String::as_str) {
Some("7") => (StatusCode::NOT_FOUND, "volume id 7 not found"),
_ => (
StatusCode::SERVICE_UNAVAILABLE,
"service warming up, please retry",
),
};
(
status,
axum::Json(serde_json::json!({
"volumeOrFileId": params.get("volumeId"),
"error": error
})),
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let master_url = format!("127.0.0.1:{}", addr.port());
let client = reqwest::Client::new();
let locations = super::lookup_volume(&client, "http", &master_url, 7)
.await
.expect("missing volume is not a lookup failure");
assert!(locations.is_empty());
let err = super::lookup_volume(&client, "http", &master_url, 8)
.await
.expect_err("master failure is a lookup failure");
assert!(err.contains("warming up"));
server.abort();
}
/// A body streaming `data_size` bytes of `file` from offset 0.
fn file_streaming_body(
file: &std::fs::File,
@@ -5392,6 +5515,7 @@ mod tests {
});
let info = ProxyRequestInfo {
method: Method::GET,
original_headers: HeaderMap::new(),
original_query: String::new(),
path: "/3,01637037d6".to_string(),
@@ -5424,6 +5548,183 @@ mod tests {
server.abort();
}
/// Go now forwards the inbound method (volume_server_handlers_read.go), so
/// a client HEAD reaches the replica as HEAD, not GET.
#[tokio::test]
async fn test_proxy_request_preserves_method() {
use axum::{Router, routing::get};
use std::sync::Arc as StdArc;
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
let head_count = StdArc::new(AtomicUsize::new(0));
let get_count = StdArc::new(AtomicUsize::new(0));
let app = {
let head_count = head_count.clone();
let get_count = get_count.clone();
Router::new().route(
"/{*path}",
get(move |request: axum::extract::Request| {
let head_count = head_count.clone();
let get_count = get_count.clone();
async move {
if *request.method() == Method::HEAD {
head_count.fetch_add(1, AtomicOrdering::SeqCst);
} else {
get_count.fetch_add(1, AtomicOrdering::SeqCst);
}
StatusCode::OK
}
}),
)
};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let target = VolumeLocation {
url: addr.to_string(),
public_url: String::new(),
grpc_port: 0,
read_only: false,
read_only_can_delete: false,
};
let state = streaming_test_state();
for method in [Method::HEAD, Method::GET] {
let info = ProxyRequestInfo {
method,
original_headers: HeaderMap::new(),
original_query: String::new(),
path: "/3,01637037d6".to_string(),
vid_str: "3".to_string(),
fid_str: "01637037d6".to_string(),
};
let response = proxy_request(&state, &info, &target).await;
assert_eq!(response.status(), StatusCode::OK);
}
assert_eq!(head_count.load(AtomicOrdering::SeqCst), 1);
assert_eq!(get_count.load(AtomicOrdering::SeqCst), 1);
server.abort();
}
/// End-to-end headers like Range reach the replica; hop-by-hop headers do
/// not — a forwarded Content-Length or Host describes this hop, not the
/// replica's, and a lying Content-Length stalls the peer's body read.
#[tokio::test]
async fn test_proxy_request_drops_hop_by_hop_headers() {
use axum::{Router, routing::get};
use std::sync::Arc as StdArc;
use std::sync::Mutex;
let seen = StdArc::new(Mutex::new(Vec::new()));
let app = {
let seen = seen.clone();
Router::new().route(
"/{*path}",
get(move |request: axum::extract::Request| {
let seen = seen.clone();
async move {
let headers: Vec<(String, String)> = request
.headers()
.iter()
.map(|(k, v)| {
(k.as_str().to_string(), v.to_str().unwrap_or("").to_string())
})
.collect();
seen.lock().unwrap().push(headers);
request
.headers()
.get(header::RANGE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string()
}
}),
)
};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let target = VolumeLocation {
url: addr.to_string(),
public_url: String::new(),
grpc_port: 0,
read_only: false,
read_only_can_delete: false,
};
let mut original_headers = HeaderMap::new();
original_headers.insert(header::RANGE, header::HeaderValue::from_static("bytes=0-3"));
original_headers.insert(
header::HOST,
header::HeaderValue::from_static("client-facing:8080"),
);
original_headers.insert(
header::CONTENT_LENGTH,
header::HeaderValue::from_static("999"),
);
original_headers.insert(
header::TRANSFER_ENCODING,
header::HeaderValue::from_static("chunked"),
);
original_headers.insert(
header::CONNECTION,
header::HeaderValue::from_static("keep-alive, x-hop-marker"),
);
original_headers.insert(
"x-hop-marker",
header::HeaderValue::from_static("private"),
);
original_headers.insert(
"proxy-authorization",
header::HeaderValue::from_static("Basic cHJveHk6c2VjcmV0"),
);
let info = ProxyRequestInfo {
method: Method::GET,
original_headers,
original_query: String::new(),
path: "/3,01637037d6".to_string(),
vid_str: "3".to_string(),
fid_str: "01637037d6".to_string(),
};
let response = proxy_request(&streaming_test_state(), &info, &target).await;
assert_eq!(response.status(), StatusCode::OK);
let seen = seen.lock().unwrap();
let headers = seen.first().expect("peer saw a request");
assert!(
headers
.iter()
.any(|(n, v)| n == "range" && v == "bytes=0-3"),
"Range was not forwarded: {headers:?}"
);
for dropped in [
"content-length",
"transfer-encoding",
"connection",
"x-hop-marker",
"proxy-authorization",
] {
assert!(
!headers.iter().any(|(n, _)| n == dropped),
"hop-by-hop header {dropped} reached the replica: {headers:?}"
);
}
// The peer must see its own host, not the client's.
let host = headers
.iter()
.find(|(n, _)| n == "host")
.map(|(_, v)| v.as_str())
.expect("replica request has a host header");
assert_eq!(host, addr.to_string());
server.abort();
}
fn streaming_test_state() -> Arc<VolumeServerState> {
use crate::storage::needle_map::NeedleMapKind;
use crate::storage::store::Store;
@@ -7182,7 +7483,7 @@ mod tests {
header::RANGE,
header::HeaderValue::from_bytes(b"bytes=0-1\xff").unwrap(),
);
let info = build_proxy_request_info(&path, &headers, "").unwrap();
let info = build_proxy_request_info(&path, &headers, "", &Method::GET).unwrap();
let resp = proxy_request(&proxy_state, &info, &target).await;
assert_eq!(resp.status(), StatusCode::RANGE_NOT_SATISFIABLE);
+13 -3
View File
@@ -79,7 +79,11 @@ func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId s
}
func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) {
results, err := LookupVolumeIds(masterFn, grpcDialOption, []string{vid})
return LookupVolumeIdWithContext(context.Background(), masterFn, grpcDialOption, vid)
}
func LookupVolumeIdWithContext(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) {
results, err := LookupVolumeIdsWithContext(ctx, masterFn, grpcDialOption, []string{vid})
return results[vid], err
}
@@ -90,6 +94,12 @@ func InvalidateVolumeIdLocationCache(vid string) {
// LookupVolumeIds find volume locations by cache and actual lookup
func LookupVolumeIds(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids []string, useCache ...bool) (map[string]*LookupResult, error) {
return LookupVolumeIdsWithContext(context.Background(), masterFn, grpcDialOption, vids, useCache...)
}
// LookupVolumeIdsWithContext is LookupVolumeIds bound to ctx, so a caller's
// cancellation (e.g. a dropped inbound request) also stops the master query.
func LookupVolumeIdsWithContext(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids []string, useCache ...bool) (map[string]*LookupResult, error) {
ret := make(map[string]*LookupResult)
var unknown_vids []string
cacheLocations := len(useCache) == 0 || useCache[0]
@@ -112,12 +122,12 @@ func LookupVolumeIds(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids
//only query unknown_vids
err := WithMasterServerClient(context.Background(), false, masterFn(context.Background()), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
err := WithMasterServerClient(ctx, false, masterFn(ctx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
req := &master_pb.LookupVolumeRequest{
VolumeOrFileIds: unknown_vids,
}
resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
resp, grpcErr := masterClient.LookupVolume(ctx, req)
if grpcErr != nil {
return grpcErr
}
+37 -5
View File
@@ -12,6 +12,7 @@ import (
"net/http"
"net/url"
"path/filepath"
"slices"
"strconv"
"strings"
"sync/atomic"
@@ -64,17 +65,28 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re
w.WriteHeader(http.StatusBadRequest)
return
}
lookupResult, err := operation.LookupVolumeId(vs.GetMaster, vs.grpcDialOption, volumeId.String())
if err != nil || len(lookupResult.Locations) <= 0 {
lookupResult, err := operation.LookupVolumeIdWithContext(r.Context(), vs.GetMaster, vs.grpcDialOption, volumeId.String())
if err != nil && vs.ReadMode == "proxy" {
if r.Context().Err() != nil {
return
}
glog.V(0).InfofCtx(r.Context(), "lookup error: %v %s", err, r.URL.Path)
InternalError(w, r, nil)
return
}
if err != nil || lookupResult == nil || len(lookupResult.Locations) <= 0 {
glog.V(0).Infoln("lookup error:", err, r.URL.Path)
stats.VolumeServerFileReadInvalidNeedles.Inc()
NotFound(w)
return
}
if len(lookupResult.Locations) >= 2 {
rand.Shuffle(len(lookupResult.Locations), func(i, j int) {
lookupResult.Locations[i], lookupResult.Locations[j] = lookupResult.Locations[j], lookupResult.Locations[i]
// cached locations are shared between lookups, so shuffle a copy
locations := slices.Clone(lookupResult.Locations)
rand.Shuffle(len(locations), func(i, j int) {
locations[i], locations[j] = locations[j], locations[i]
})
lookupResult.Locations = locations
}
var targetUrl *url.URL
location := fmt.Sprintf("%s:%d", vs.store.Ip, vs.store.Port)
@@ -100,13 +112,30 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re
query := r.URL.Query()
query.Set(reqIsProxied, "true")
r.URL.RawQuery = query.Encode()
request, err := http.NewRequest(http.MethodGet, r.URL.String(), nil)
request, err := http.NewRequestWithContext(r.Context(), r.Method, r.URL.String(), nil)
if err != nil {
glog.V(0).Infof("failed to instance http request of url %s: %v", redactedURL(r), err)
InternalError(w, r, nil)
return
}
// hop-by-hop headers describe this hop, not the replica's; Connection
// also nominates per-request fields to drop (RFC 9110 7.6.1)
hopByHop := map[string]bool{
"Connection": true, "Keep-Alive": true, "Te": true, "Trailer": true,
"Upgrade": true, "Host": true, "Content-Length": true, "Transfer-Encoding": true,
"Proxy-Connection": true, "Proxy-Authenticate": true, "Proxy-Authorization": true,
}
for _, v := range r.Header.Values("Connection") {
for _, token := range strings.Split(v, ",") {
if name := http.CanonicalHeaderKey(strings.TrimSpace(token)); name != "" {
hopByHop[name] = true
}
}
}
for k, vv := range r.Header {
if hopByHop[http.CanonicalHeaderKey(k)] {
continue
}
for _, v := range vv {
request.Header.Add(k, v)
}
@@ -114,6 +143,9 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re
response, err := util_http.GetGlobalHttpClient().Do(request)
if err != nil {
if r.Context().Err() != nil {
return
}
stats.VolumeServerHandlerCounter.WithLabelValues(stats.FailedReadProxyReq).Inc()
glog.V(0).Infof("request remote url %s: %v", redactedURL(r), err)
InternalError(w, r, nil)
@@ -1,12 +1,25 @@
package weed_server
import (
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
func TestInternalErrorWritesTheReadError(t *testing.T) {
@@ -96,3 +109,296 @@ func TestNotModifiedChecksIfNoneMatchFirst(t *testing.T) {
})
}
}
type proxyTestMaster struct {
master_pb.UnimplementedSeaweedServer
lookup func(context.Context, *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error)
}
func (m *proxyTestMaster) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
return m.lookup(ctx, req)
}
func newProxyTestHTTPServer(t *testing.T, handler http.Handler) *httptest.Server {
t.Helper()
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
server := &httptest.Server{Listener: listener, Config: &http.Server{Handler: handler}}
server.Start()
t.Cleanup(server.Close)
return server
}
type proxyTestResponseWriter struct {
http.ResponseWriter
statusWritten chan struct{}
}
func (w proxyTestResponseWriter) WriteHeader(status int) {
w.ResponseWriter.WriteHeader(status)
w.statusWritten <- struct{}{}
}
func newProxyTestVolumeServer(t *testing.T, volumeID string, locations []*master_pb.Location, lookupErr error) *VolumeServer {
t.Helper()
operation.InvalidateVolumeIdLocationCache(volumeID)
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
masterAddress := pb.NewServerAddress("127.0.0.1", 9333, listener.Addr().(*net.TCPAddr).Port)
masterServer := grpc.NewServer()
master_pb.RegisterSeaweedServer(masterServer, &proxyTestMaster{lookup: func(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
if lookupErr != nil {
return nil, lookupErr
}
if len(req.VolumeOrFileIds) == 0 {
return nil, errors.New("lookup request contains no volume IDs")
}
return &master_pb.LookupVolumeResponse{VolumeIdLocations: []*master_pb.LookupVolumeResponse_VolumeIdLocation{{
VolumeOrFileId: req.VolumeOrFileIds[0],
Locations: locations,
}}}, nil
}})
go func() {
_ = masterServer.Serve(listener)
}()
vs := &VolumeServer{
store: &storage.Store{Ip: "192.0.2.10", Port: 8080},
ReadMode: "proxy",
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
}
vs.setCurrentMaster(masterAddress)
t.Cleanup(func() {
operation.InvalidateVolumeIdLocationCache(volumeID)
pb.InvalidateGrpcConnection(masterAddress.ToGrpcAddress())
masterServer.Stop()
})
return vs
}
func TestProxyReqToTargetServerPreservesMethodAndResponse(t *testing.T) {
const volumeID = "849300101"
type observedRequest struct {
method string
rangeHeader string
ifNoneMatch string
ifModifiedAt string
header http.Header
}
observed := make(chan observedRequest, 2)
peer := newProxyTestHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
observed <- observedRequest{
method: r.Method,
rangeHeader: r.Header.Get("Range"),
ifNoneMatch: r.Header.Get("If-None-Match"),
ifModifiedAt: r.Header.Get("If-Modified-Since"),
header: r.Header.Clone(),
}
w.Header().Set("ETag", `"peer-etag"`)
w.Header().Set("Content-Range", "bytes 0-3/10")
w.Header().Set("X-Peer-Header", "preserved")
w.WriteHeader(http.StatusPartialContent)
_, _ = io.WriteString(w, "data")
}))
remoteAddress := strings.TrimPrefix(peer.URL, "http://")
vs := newProxyTestVolumeServer(t, volumeID, []*master_pb.Location{{Url: remoteAddress}}, nil)
for _, method := range []string{http.MethodGet, http.MethodHead} {
t.Run(method, func(t *testing.T) {
r := httptest.NewRequest(method, "/"+volumeID+",abcdef", nil)
r.Header.Set("Range", "bytes=0-3")
r.Header.Set("If-None-Match", `"client-etag"`)
r.Header.Set("If-Modified-Since", "Wed, 21 Oct 2015 07:28:00 GMT")
r.Header.Set("Connection", "keep-alive, X-Hop-Marker")
r.Header.Set("Keep-Alive", "timeout=5")
r.Header.Set("Upgrade", "h2c")
r.Header.Set("Te", "trailers")
r.Header.Set("X-Hop-Marker", "private")
r.Header.Set("Proxy-Connection", "keep-alive")
r.Header.Set("Proxy-Authorization", "Basic cHJveHk6c2VjcmV0")
w := httptest.NewRecorder()
vs.proxyReqToTargetServer(w, r)
var got observedRequest
select {
case got = <-observed:
case <-time.After(2 * time.Second):
t.Fatalf("peer did not receive the proxied request; proxy status %d", w.Code)
}
if got.method != method {
t.Errorf("peer received method %q, want %q", got.method, method)
}
if got.rangeHeader != "bytes=0-3" || got.ifNoneMatch != `"client-etag"` || got.ifModifiedAt != "Wed, 21 Oct 2015 07:28:00 GMT" {
t.Errorf("peer received conditional/range headers %+v", got)
}
for _, dropped := range []string{"Connection", "Keep-Alive", "Upgrade", "Te", "X-Hop-Marker", "Proxy-Connection", "Proxy-Authorization"} {
if got.header.Get(dropped) != "" {
t.Errorf("hop-by-hop header %s reached the replica: %v", dropped, got.header)
}
}
if w.Code != http.StatusPartialContent {
t.Errorf("status %d, want %d", w.Code, http.StatusPartialContent)
}
if w.Header().Get("ETag") != `"peer-etag"` || w.Header().Get("Content-Range") != "bytes 0-3/10" || w.Header().Get("X-Peer-Header") != "preserved" {
t.Errorf("peer response headers were not preserved: %v", w.Header())
}
wantBody := "data"
if method == http.MethodHead {
wantBody = ""
}
if gotBody := w.Body.String(); gotBody != wantBody {
t.Errorf("response body %q, want %q", gotBody, wantBody)
}
})
}
}
func TestProxyReqToTargetServerCancelsUpstreamWithInboundRequest(t *testing.T) {
const volumeID = "849300102"
peerStarted := make(chan struct{})
peerCanceled := make(chan struct{})
releasePeer := make(chan struct{})
var releasePeerOnce sync.Once
peer := newProxyTestHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(peerStarted)
select {
case <-r.Context().Done():
close(peerCanceled)
case <-releasePeer:
}
}))
defer releasePeerOnce.Do(func() { close(releasePeer) })
remoteAddress := strings.TrimPrefix(peer.URL, "http://")
vs := newProxyTestVolumeServer(t, volumeID, []*master_pb.Location{{Url: remoteAddress}}, nil)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
r := httptest.NewRequest(http.MethodGet, "/"+volumeID+",abcdef", nil).WithContext(ctx)
w := httptest.NewRecorder()
proxyDone := make(chan struct{})
go func() {
vs.proxyReqToTargetServer(w, r)
close(proxyDone)
}()
select {
case <-peerStarted:
case <-time.After(2 * time.Second):
t.Fatal("peer did not receive the proxied request")
}
cancel()
select {
case <-peerCanceled:
case <-time.After(time.Second):
t.Error("canceling the inbound request did not cancel the peer request")
}
releasePeerOnce.Do(func() { close(releasePeer) })
select {
case <-proxyDone:
case <-time.After(2 * time.Second):
t.Error("proxy handler did not finish after inbound cancellation")
}
}
func TestProxyReqToTargetServerCancelsUpstreamBodyReadWithInboundRequest(t *testing.T) {
const volumeID = "849300106"
peerCanceled := make(chan struct{})
releasePeer := make(chan struct{})
var releasePeerOnce sync.Once
peer := newProxyTestHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.(http.Flusher).Flush()
select {
case <-r.Context().Done():
close(peerCanceled)
case <-releasePeer:
}
}))
defer releasePeerOnce.Do(func() { close(releasePeer) })
remoteAddress := strings.TrimPrefix(peer.URL, "http://")
vs := newProxyTestVolumeServer(t, volumeID, []*master_pb.Location{{Url: remoteAddress}}, nil)
r := httptest.NewRequest(http.MethodGet, "/"+volumeID+",abcdef", nil)
ctx, cancel := context.WithCancel(r.Context())
r = r.WithContext(ctx)
recorder := httptest.NewRecorder()
statusWritten := make(chan struct{}, 1)
w := proxyTestResponseWriter{ResponseWriter: recorder, statusWritten: statusWritten}
proxyDone := make(chan struct{})
go func() {
vs.proxyReqToTargetServer(w, r)
close(proxyDone)
}()
select {
case <-statusWritten:
case <-time.After(2 * time.Second):
cancel()
t.Fatal("proxy did not receive the peer response headers")
}
cancel()
select {
case <-peerCanceled:
case <-time.After(time.Second):
t.Error("canceling the inbound request did not cancel the peer response body")
}
releasePeerOnce.Do(func() { close(releasePeer) })
select {
case <-proxyDone:
case <-time.After(2 * time.Second):
t.Error("proxy handler did not finish reading after inbound cancellation")
}
}
func TestProxyReqToTargetServerDistinguishesLookupFailureFromNoLocation(t *testing.T) {
t.Run("cancelled request skips the lookup error response", func(t *testing.T) {
const volumeID = "849300107"
vs := newProxyTestVolumeServer(t, volumeID, nil, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
w := httptest.NewRecorder()
vs.proxyReqToTargetServer(w, httptest.NewRequest(http.MethodGet, "/"+volumeID+",abcdef", nil).WithContext(ctx))
if w.Code != http.StatusOK {
t.Errorf("status %d, want no response written for a cancelled request", w.Code)
}
})
t.Run("master lookup failure is internal error", func(t *testing.T) {
const volumeID = "849300103"
vs := newProxyTestVolumeServer(t, volumeID, nil, status.Error(codes.Unavailable, "master unavailable"))
w := httptest.NewRecorder()
vs.proxyReqToTargetServer(w, httptest.NewRequest(http.MethodGet, "/"+volumeID+",abcdef", nil))
if w.Code != http.StatusInternalServerError {
t.Errorf("status %d, want %d", w.Code, http.StatusInternalServerError)
}
})
for _, tc := range []struct {
name string
locations []*master_pb.Location
}{
{name: "lookup returned no replica locations"},
{name: "only local replica is not a usable proxy target", locations: []*master_pb.Location{{Url: "192.0.2.10:8080"}}},
} {
t.Run(tc.name, func(t *testing.T) {
volumeID := "84930010"
if tc.name == "lookup returned no replica locations" {
volumeID += "4"
} else {
volumeID += "5"
}
vs := newProxyTestVolumeServer(t, volumeID, tc.locations, nil)
w := httptest.NewRecorder()
vs.proxyReqToTargetServer(w, httptest.NewRequest(http.MethodGet, "/"+volumeID+",abcdef", nil))
if w.Code != http.StatusNotFound {
t.Errorf("status %d, want %d", w.Code, http.StatusNotFound)
}
})
}
}