mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
operation: reassign an upload after a proven-expired volume JWT (#12729)
* operation: reassign an upload after a proven-expired volume JWT * seaweed-volume: mark an expired write JWT refused before append Mirror the Go volume server: a multipart POST whose write JWT is signed with the current key, names this fid and fails only on exp gets X-Seaweed-Write-Outcome: expired-jwt-before-append on its 401, so the uploader reassigns instead of failing. Both checks use one guard snapshot, so a key rotation between them cannot drop the marker. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: keep same-FID retry for filer-proxy expiries, snapshot the write signing key, and make the RetryForever budget test fail fast * operation: re-mint the filer credential before a proxy same-FID expiry retry * operation: make a proxy marked-expiry terminal in the outer retry too, not only the inner same-FID loop --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
11 files changed
+1215
-29
No files matched your search
@@ -106,6 +106,39 @@ pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims,
|
||||
Ok(data.claims)
|
||||
}
|
||||
|
||||
/// Reports a signed, fid-bound write token whose only failure is expiry. It
|
||||
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
|
||||
pub fn is_expired_volume_jwt(signing_key: &SigningKey, token: &str, expected_fid: &str) -> bool {
|
||||
match decode_jwt(signing_key, token) {
|
||||
Err(JwtError::Jwt(e))
|
||||
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
|
||||
_ => return false,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ExpiredClaims {
|
||||
fid: Option<String>,
|
||||
iat: Option<u64>,
|
||||
}
|
||||
let mut validation = Validation::new(Algorithm::HS256);
|
||||
validation.required_spec_claims.clear();
|
||||
validation.validate_exp = false;
|
||||
validation.validate_nbf = true;
|
||||
validation.leeway = 0;
|
||||
let Ok(data) = decode::<ExpiredClaims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(&signing_key.0),
|
||||
&validation,
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Guard
|
||||
// ============================================================================
|
||||
@@ -410,6 +443,53 @@ mod tests {
|
||||
assert!(decode_jwt(&key2, &token).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_expired_volume_jwt_classifies_expiry_only() {
|
||||
let key = SigningKey::from_string("write-key");
|
||||
let fid = "3,01637037d6";
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let sign = |claims: serde_json::Value, key: &SigningKey| {
|
||||
encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&claims,
|
||||
&EncodingKey::from_secret(&key.0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
|
||||
assert!(is_expired_volume_jwt(&key, &expired, fid));
|
||||
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7"));
|
||||
assert!(!is_expired_volume_jwt(
|
||||
&SigningKey::from_string("other-key"),
|
||||
&expired,
|
||||
fid
|
||||
));
|
||||
assert!(!is_expired_volume_jwt(&SigningKey(vec![]), &expired, fid));
|
||||
let immature = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &immature, fid));
|
||||
let future_issued = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &future_issued, fid));
|
||||
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
|
||||
assert!(!is_expired_volume_jwt(&key, &live, fid));
|
||||
let unsigned = format!(
|
||||
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
|
||||
base64::Engine::encode(
|
||||
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
|
||||
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
|
||||
)
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &unsigned, fid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_empty_whitelist() {
|
||||
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
|
||||
@@ -483,6 +483,18 @@ fn extract_file_id(path: &str) -> String {
|
||||
format!("{},{}", parsed.vid, fid)
|
||||
}
|
||||
|
||||
fn is_multipart_form_data(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.split(';').next())
|
||||
.is_some_and(|media_type| {
|
||||
media_type
|
||||
.trim()
|
||||
.eq_ignore_ascii_case("multipart/form-data")
|
||||
})
|
||||
}
|
||||
|
||||
fn streaming_chunk_size(read_buffer_size_bytes: usize, data_size: usize) -> usize {
|
||||
std::cmp::min(
|
||||
read_buffer_size_bytes.max(DEFAULT_STREAMING_CHUNK_SIZE),
|
||||
@@ -2777,14 +2789,28 @@ pub async fn post_handler(
|
||||
// JWT check for writes
|
||||
let file_id = extract_file_id(&path);
|
||||
let token = extract_jwt(&headers, request.uri());
|
||||
if state
|
||||
.guard
|
||||
.read()
|
||||
.unwrap()
|
||||
.check_jwt_for_file(token.as_deref(), &file_id, true)
|
||||
.is_err()
|
||||
{
|
||||
return json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
|
||||
let expired_jwt = {
|
||||
let guard = state.guard.read().unwrap();
|
||||
match guard.check_jwt_for_file(token.as_deref(), &file_id, true) {
|
||||
Ok(()) => None,
|
||||
Err(_) => Some(
|
||||
is_multipart_form_data(&headers)
|
||||
&& token.as_deref().is_some_and(|token| {
|
||||
crate::security::is_expired_volume_jwt(&guard.signing_key, token, &file_id)
|
||||
}),
|
||||
),
|
||||
}
|
||||
};
|
||||
if let Some(expired_jwt) = expired_jwt {
|
||||
let mut response =
|
||||
json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
|
||||
if expired_jwt {
|
||||
response.headers_mut().insert(
|
||||
"x-seaweed-write-outcome",
|
||||
header::HeaderValue::from_static("expired-jwt-before-append"),
|
||||
);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
// Upload throttling: check inflight bytes against limit
|
||||
@@ -5465,6 +5491,83 @@ mod tests {
|
||||
assert_eq!(extract_jwt(&headers, &uri), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_expired_write_jwt_is_marked_before_append() {
|
||||
use crate::security::SigningKey;
|
||||
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
|
||||
use tower::ServiceExt;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let state = volume_test_state(&tmp);
|
||||
let key = SigningKey(b"write-secret".to_vec());
|
||||
state.guard.write().unwrap().signing_key = key.clone();
|
||||
let volume = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
let before = volume.dat_file_size().unwrap();
|
||||
let fid = format!("1,{:x}{TEST_COOKIE:08x}", 0x6e7a_0c09u64);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let sign = |key: &[u8]| {
|
||||
encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&serde_json::json!({"fid": fid, "exp": now - 10}),
|
||||
&EncodingKey::from_secret(key),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let boundary = "expiredjwtboundary";
|
||||
let body = format!(
|
||||
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a\"\r\n\r\nexpired-bytes\r\n--{boundary}--\r\n"
|
||||
);
|
||||
for (token, content_type, marked) in [
|
||||
(
|
||||
sign(&key.0),
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
true,
|
||||
),
|
||||
(
|
||||
sign(b"other-secret"),
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
false,
|
||||
),
|
||||
(sign(&key.0), "application/octet-stream".to_string(), false),
|
||||
] {
|
||||
let response = super::super::volume_server::build_admin_router(state.clone())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(format!("/{fid}"))
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.header(header::CONTENT_TYPE, &content_type)
|
||||
.body(Body::from(body.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"{content_type}"
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-seaweed-write-outcome")
|
||||
.map(|value| value.to_str().unwrap()),
|
||||
marked.then_some("expired-jwt-before-append"),
|
||||
"{content_type}"
|
||||
);
|
||||
}
|
||||
assert_eq!(volume.dat_file_size().unwrap(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_range_single() {
|
||||
let data = b"hello world";
|
||||
|
||||
@@ -131,6 +131,7 @@ var (
|
||||
type uploadStatusError struct {
|
||||
StatusCode int
|
||||
refusedBeforeAppend bool
|
||||
expiredBeforeAppend bool
|
||||
stalled bool // the stall watchdog cancelled this attempt's request
|
||||
err error
|
||||
}
|
||||
@@ -149,7 +150,7 @@ func ShouldReassignUpload(err error) bool {
|
||||
}
|
||||
var se *uploadStatusError
|
||||
if errors.As(err, &se) {
|
||||
return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend)
|
||||
return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend) || (se.StatusCode == http.StatusUnauthorized && se.expiredBeforeAppend)
|
||||
}
|
||||
return strings.Contains(err.Error(), "transport")
|
||||
}
|
||||
@@ -243,17 +244,37 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho
|
||||
return err
|
||||
}
|
||||
|
||||
// A credential can expire while the first HTTP request is queued. Bound
|
||||
// expiry-specific reassignment even when unrelated transient retries are
|
||||
// intentionally unbounded. The third marked refusal is final.
|
||||
expiryRefusals := 0
|
||||
shouldRetry := func(err error) bool {
|
||||
var statusErr *uploadStatusError
|
||||
if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend {
|
||||
// A proxy upload already exhausted its bounded same-FID retries in
|
||||
// retriedUploadData (with a fresh filer credential each time); the
|
||||
// filer owns the FID, so a reassignment here is wrong, and counting
|
||||
// exhausted groups would also inflate the refusal budget. Make it
|
||||
// terminal; only direct uploads reassign.
|
||||
if util_http.IsProxyChunkUrl(uploadOption.UploadUrl) {
|
||||
return false
|
||||
}
|
||||
expiryRefusals++
|
||||
return expiryRefusals < 3
|
||||
}
|
||||
return ShouldReassignUpload(err)
|
||||
}
|
||||
if uploadOption.RetryForever {
|
||||
util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
|
||||
glog.V(0).Infof("upload content: %v", err)
|
||||
var statusErr *uploadStatusError
|
||||
if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusConflict {
|
||||
return ShouldReassignUpload(err)
|
||||
if errors.As(err, &statusErr) && (statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend)) {
|
||||
return shouldRetry(err)
|
||||
}
|
||||
return true
|
||||
})
|
||||
} else {
|
||||
err = util.RetryOnError("uploadWithRetry", ShouldReassignUpload, doUploadFunc)
|
||||
err = util.RetryOnError("uploadWithRetry", shouldRetry, doUploadFunc)
|
||||
}
|
||||
|
||||
return
|
||||
@@ -380,9 +401,26 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *prepa
|
||||
}
|
||||
var statusErr *uploadStatusError
|
||||
if errors.As(err, &statusErr) {
|
||||
if statusErr.StatusCode == http.StatusConflict {
|
||||
// A marked-expiry 401 is terminal here only for a direct volume
|
||||
// upload: its assignment JWT is spent, so recovery needs a fresh
|
||||
// assignment (uploadWithRetryData). A filer-proxy upload instead
|
||||
// mints a fresh volume JWT on each forwarded request, so a bounded
|
||||
// same-FID retry is enough and reassignment is both unnecessary and
|
||||
// wrong (the proxy owns the FID). Keep that same-FID retry for it.
|
||||
if statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && !util_http.IsProxyChunkUrl(option.UploadUrl)) {
|
||||
return nil, err
|
||||
}
|
||||
if statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && util_http.IsProxyChunkUrl(option.UploadUrl) {
|
||||
// The caller's filer credential was minted once and may have
|
||||
// expired alongside the volume JWT (both default to 10s), so
|
||||
// re-mint it before replaying the same FID; otherwise the filer
|
||||
// rejects the retry with an unmarked 401 and recovery stops.
|
||||
if option.FilerJwt != nil {
|
||||
option.Jwt = option.FilerJwt(true)
|
||||
} else {
|
||||
option.Jwt = security.EncodedJwt(util_http.JwtForFilerServer(true))
|
||||
}
|
||||
}
|
||||
if statusErr.stalled {
|
||||
option.stallWatchdogDisarmed = true
|
||||
}
|
||||
@@ -720,26 +758,27 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
|
||||
respStallTimer := time.AfterFunc(uploadStallMinWait, stallFired)
|
||||
defer respStallTimer.Stop()
|
||||
}
|
||||
refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get("X-Seaweed-Write-Outcome") == "refused-before-append"
|
||||
refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeRefusedBeforeAppend
|
||||
expiredBeforeAppend := resp.StatusCode == http.StatusUnauthorized && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeExpiredJwtBeforeAppend
|
||||
resp_body, ra_err := io.ReadAll(resp.Body)
|
||||
uploadDone.Store(true)
|
||||
if ra_err != nil {
|
||||
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)}
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)}
|
||||
}
|
||||
|
||||
unmarshal_err := json.Unmarshal(resp_body, &ret)
|
||||
if unmarshal_err != nil {
|
||||
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
|
||||
glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body))
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)}
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)}
|
||||
}
|
||||
if resp.StatusCode == http.StatusConflict && ret.Error == "" {
|
||||
if (resp.StatusCode == http.StatusConflict || expiredBeforeAppend) && ret.Error == "" {
|
||||
ret.Error = http.StatusText(resp.StatusCode)
|
||||
}
|
||||
if ret.Error != "" {
|
||||
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)}
|
||||
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)}
|
||||
}
|
||||
if bps := uploadObservedBps(bodySize, movedIn); bps > 0 {
|
||||
uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps})
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUploadExpiryResponseContract(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
marker, body string
|
||||
want bool
|
||||
calls int
|
||||
}{
|
||||
{"marked", 401, "expired-jwt-before-append", `{"error":"wrong jwt"}`, true, 1},
|
||||
{"marked_empty_error", 401, "expired-jwt-before-append", `{}`, true, 1},
|
||||
{"marked_bad_json", 401, "expired-jwt-before-append", `{`, true, 1},
|
||||
{"unmarked", 401, "", `{"error":"wrong jwt"}`, false, 3},
|
||||
{"wrong_value", 401, "expired-jwt-before-append-extra", `{"error":"wrong jwt"}`, false, 3},
|
||||
{"wrong_status", 403, "expired-jwt-before-append", `{"error":"denied"}`, false, 3},
|
||||
{"existing_409", 409, "refused-before-append", `{"error":"full"}`, true, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, tc.marker)
|
||||
w.WriteHeader(tc.status)
|
||||
_, _ = io.WriteString(w, tc.body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
u := NewUploaderWithHttpClient(srv.Client())
|
||||
_, err := u.UploadData(context.Background(), []byte("direct preassigned data"), &UploadOption{UploadUrl: srv.URL, Jwt: "disposable-preassigned"})
|
||||
require.Error(t, err)
|
||||
require.Equal(t, tc.want, ShouldReassignUpload(err))
|
||||
mu.Lock()
|
||||
actual := calls
|
||||
mu.Unlock()
|
||||
require.Equal(t, tc.calls, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadExpiryBudgetAndAssignmentFailure(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
forever, assignmentFailure bool
|
||||
}{{"bounded", false, false}, {"forever", true, false}, {"assignment_failure", false, true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
var payloads [][]byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
data, err := io.ReadAll(part)
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
payloads = append(payloads, data)
|
||||
mu.Unlock()
|
||||
// Terminal fallback: if the expiry budget regressed and did not
|
||||
// stop at 3, end the run with an unmarked 409 so this test fails
|
||||
// here instead of looping to the package timeout under RetryForever.
|
||||
if n > 3 {
|
||||
w.WriteHeader(409)
|
||||
_, _ = io.WriteString(w, `{"error":"full"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
u := NewUploaderWithHttpClient(srv.Client())
|
||||
assigns := 0
|
||||
sentinel := errors.New("synthetic assignment failure")
|
||||
_, _, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
|
||||
assigns++
|
||||
if tc.assignmentFailure && assigns == 2 {
|
||||
return "", "", "", false, sentinel
|
||||
}
|
||||
return fmt.Sprintf("1,%08x00001234", assigns), strings.TrimPrefix(srv.URL, "http://"), "disposable-per-assignment", false, nil
|
||||
}, &UploadOption{Cipher: true, MimeType: "application/octet-stream", RetryForever: tc.forever}, bytes.Repeat([]byte("compressible data"), 1024))
|
||||
require.Error(t, err)
|
||||
mu.Lock()
|
||||
actual := calls
|
||||
got := append([][]byte(nil), payloads...)
|
||||
mu.Unlock()
|
||||
if tc.assignmentFailure {
|
||||
require.ErrorIs(t, err, sentinel)
|
||||
require.Equal(t, 2, assigns)
|
||||
require.Equal(t, 1, actual)
|
||||
} else {
|
||||
require.Equal(t, 3, assigns)
|
||||
require.Equal(t, 3, actual)
|
||||
// The run must end on the 3rd marked 401 (the budget), not the
|
||||
// 409 terminal fallback above.
|
||||
var finalErr *uploadStatusError
|
||||
require.ErrorAs(t, err, &finalErr)
|
||||
require.Equal(t, http.StatusUnauthorized, finalErr.StatusCode)
|
||||
require.True(t, finalErr.expiredBeforeAppend)
|
||||
}
|
||||
require.NotEmpty(t, got)
|
||||
for _, data := range got {
|
||||
require.Equal(t, got[0], data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A filer-proxy upload must retry a marked-expiry 401 on the SAME FID -- the
|
||||
// proxy mints a fresh volume JWT per forwarded request, so a same-FID replay
|
||||
// recovers without a reassignment it does not own.
|
||||
func TestUploadExpiryProxyRetriesSameFid(t *testing.T) {
|
||||
const fid = "3,01637037d6"
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
var seenFids []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
seenFids = append(seenFids, r.URL.Query().Get(util_http.ProxyChunkIdParam))
|
||||
mu.Unlock()
|
||||
if n == 1 {
|
||||
// The proxy-minted volume JWT expired in transit on the first try.
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"size":5}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid
|
||||
result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: proxyURL})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint32(5), result.Size)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
require.Equal(t, 2, calls)
|
||||
// Both forwarded requests used the same FID: a proxy upload never reassigns.
|
||||
require.Equal(t, []string{fid, fid}, seenFids)
|
||||
}
|
||||
|
||||
// With filer signing enabled, the caller's filer credential can expire together
|
||||
// with the volume JWT. The proxy same-FID retry must carry a freshly minted
|
||||
// filer credential, not replay the stale one (which the filer would reject with
|
||||
// an unmarked 401 that stops recovery).
|
||||
func TestUploadExpiryProxyRefreshesFilerJwtOnRetry(t *testing.T) {
|
||||
const fid = "3,01637037d6"
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
var auths []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
auths = append(auths, r.Header.Get("Authorization"))
|
||||
mu.Unlock()
|
||||
if n == 1 {
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"size":5}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid
|
||||
minted := 0
|
||||
option := &UploadOption{UploadUrl: proxyURL, Jwt: "stale-filer-jwt", FilerJwt: func(isWrite bool) security.EncodedJwt {
|
||||
minted++
|
||||
return security.EncodedJwt(fmt.Sprintf("fresh-filer-jwt-%d", minted))
|
||||
}}
|
||||
result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), option)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint32(5), result.Size)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
require.Equal(t, 2, calls)
|
||||
require.Equal(t, 1, minted)
|
||||
// The first request carried the stale credential; the retry re-minted one.
|
||||
require.Equal(t, "Bearer stale-filer-jwt", auths[0])
|
||||
require.Equal(t, "Bearer fresh-filer-jwt-1", auths[1])
|
||||
}
|
||||
|
||||
// A proxy upload driven through uploadWithRetryData must be terminal after its
|
||||
// bounded same-FID attempts: the outer loop must not reassign a FID the proxy
|
||||
// owns, nor count exhausted groups toward the expiry budget.
|
||||
func TestUploadExpiryProxyDoesNotReassignThroughOuterRetry(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
// The proxy keeps reporting the marked expiry (never recovers).
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
host := strings.TrimPrefix(srv.URL, "http://")
|
||||
assigns := 0
|
||||
option := &UploadOption{
|
||||
FilerJwt: func(isWrite bool) security.EncodedJwt { return "fresh-filer-jwt" },
|
||||
GenUploadUrl: func(host, fileId string) string {
|
||||
return "http://" + host + "/?" + util_http.ProxyChunkIdParam + "=" + fileId
|
||||
},
|
||||
}
|
||||
_, _, err := NewUploaderWithHttpClient(srv.Client()).uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
|
||||
assigns++
|
||||
return "3,01637037d6", host, "vol-jwt", false, nil
|
||||
}, option, []byte("plain"))
|
||||
require.Error(t, err)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// One assignment (no reassignment), and only the inner bounded same-FID
|
||||
// attempts reached the server (the outer loop did not re-enter).
|
||||
require.Equal(t, 1, assigns)
|
||||
require.Equal(t, 3, calls)
|
||||
}
|
||||
|
||||
// Generic401 still uses the old RetryForever behavior. Only the explicitly
|
||||
// marked expiry path gains the separate finite recovery budget.
|
||||
func TestUploadExpiryPreservesRetryForeverGenericDenial(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
mu.Unlock()
|
||||
if n <= 3 {
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"size":5}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
u := NewUploaderWithHttpClient(srv.Client())
|
||||
assigns := 0
|
||||
_, result, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
|
||||
assigns++
|
||||
return "1,0000000100001234", strings.TrimPrefix(srv.URL, "http://"), "disposable", false, nil
|
||||
}, &UploadOption{RetryForever: true}, []byte("plain"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint32(5), result.Size)
|
||||
require.Equal(t, 2, assigns)
|
||||
mu.Lock()
|
||||
actual := calls
|
||||
mu.Unlock()
|
||||
require.Equal(t, 4, actual)
|
||||
}
|
||||
|
||||
func TestUploadExpiryCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
cancel()
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
client := srv.Client()
|
||||
client.Timeout = 2 * time.Second
|
||||
_, err := NewUploaderWithHttpClient(client).UploadData(ctx, []byte("plain"), &UploadOption{UploadUrl: srv.URL})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
mu.Lock()
|
||||
actual := calls
|
||||
mu.Unlock()
|
||||
require.Equal(t, 1, actual)
|
||||
}
|
||||
|
||||
func TestUploadExpiryTruncatedResponsePreservesMarker(t *testing.T) {
|
||||
for _, marked := range []bool{false, true} {
|
||||
name := "unmarked"
|
||||
if marked {
|
||||
name = "marked"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
if marked {
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
}
|
||||
w.Header().Set("Content-Length", "100")
|
||||
w.WriteHeader(401)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
_, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: srv.URL})
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
var statusErr *uploadStatusError
|
||||
require.ErrorAs(t, err, &statusErr)
|
||||
require.Equal(t, 401, statusErr.StatusCode)
|
||||
require.Equal(t, marked, statusErr.expiredBeforeAppend)
|
||||
require.Equal(t, marked, ShouldReassignUpload(err))
|
||||
want := 3
|
||||
if marked {
|
||||
want = 1
|
||||
}
|
||||
mu.Lock()
|
||||
actual := calls
|
||||
mu.Unlock()
|
||||
require.Equal(t, want, actual)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -148,13 +148,17 @@ func GetJwt(r *http.Request) EncodedJwt {
|
||||
}
|
||||
|
||||
func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims) (token *jwt.Token, err error) {
|
||||
return decodeJwt(signingKey, tokenString, claims)
|
||||
}
|
||||
|
||||
func decodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims, options ...jwt.ParserOption) (token *jwt.Token, err error) {
|
||||
// check exp, nbf
|
||||
return jwt.ParseWithClaims(string(tokenString), claims, func(token *jwt.Token) (interface{}, error) {
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unknown token method")
|
||||
}
|
||||
return []byte(signingKey), nil
|
||||
})
|
||||
}, options...)
|
||||
}
|
||||
|
||||
// FilerJwtProvider signs the credential a filer's HTTP API expects. A nil
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package security
|
||||
|
||||
import (
|
||||
"errors"
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IsExpiredVolumeJwt reports a signed, FID-bound credential whose only failure
|
||||
// is expiration. It never authorizes a request; it classifies an existing denial
|
||||
// so an assignment-owning caller can obtain a fresh credential with a new FID.
|
||||
func IsExpiredVolumeJwt(key SigningKey, encoded EncodedJwt, expectedFid string) bool {
|
||||
return isExpiredVolumeJwtAt(key, encoded, expectedFid, time.Now())
|
||||
}
|
||||
|
||||
func isExpiredVolumeJwtAt(key SigningKey, encoded EncodedJwt, expectedFid string, now time.Time) bool {
|
||||
if len(key) == 0 || encoded == "" {
|
||||
return false
|
||||
}
|
||||
claims := &SeaweedFileIdClaims{}
|
||||
parsed, err := decodeJwt(key, encoded, claims, jwt.WithTimeFunc(func() time.Time { return now }))
|
||||
// ParseWithClaims verifies the HMAC signature before validating any claims.
|
||||
if parsed == nil || !errors.Is(err, jwt.ErrTokenExpired) || claims.Fid != expectedFid {
|
||||
return false
|
||||
}
|
||||
otherClaims := *claims
|
||||
otherClaims.ExpiresAt = nil
|
||||
// Do not mistake expired+not-before/issued-at errors for expiry alone. This
|
||||
// extra issued-at restriction applies only to classification, never normal
|
||||
// authorization. Keep future volume claim policies aligned here.
|
||||
return jwt.NewValidator(jwt.WithTimeFunc(func() time.Time { return now }), jwt.WithIssuedAt()).Validate(&otherClaims) == nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package security
|
||||
|
||||
import (
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestVolumeJwtExpiryOnly(t *testing.T) {
|
||||
now := time.Unix(1900000000, 0)
|
||||
key := SigningKey("disposable-volume-classifier")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
expired, nbf, iat, wrongKey, wrongFid, noExpiry bool
|
||||
want bool
|
||||
}{
|
||||
{name: "expired_only", expired: true, want: true},
|
||||
{name: "expired_future_nbf", expired: true, nbf: true},
|
||||
{name: "expired_future_iat", expired: true, iat: true},
|
||||
{name: "expired_bad_signature", expired: true, wrongKey: true},
|
||||
{name: "expired_wrong_fid", expired: true, wrongFid: true},
|
||||
{name: "valid"}, {name: "future_nbf", nbf: true}, {name: "no_expiry", noExpiry: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fid := "1,0000000100001234"
|
||||
if tc.wrongFid {
|
||||
fid = "1,0000000200001234"
|
||||
}
|
||||
exp := now.Add(time.Minute)
|
||||
if tc.expired {
|
||||
exp = now.Add(-time.Second)
|
||||
}
|
||||
claims := SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}}
|
||||
if tc.noExpiry {
|
||||
claims.ExpiresAt = nil
|
||||
}
|
||||
if tc.nbf {
|
||||
claims.NotBefore = jwt.NewNumericDate(now.Add(time.Minute))
|
||||
}
|
||||
if tc.iat {
|
||||
claims.IssuedAt = jwt.NewNumericDate(now.Add(time.Minute))
|
||||
}
|
||||
signingKey := key
|
||||
if tc.wrongKey {
|
||||
signingKey = SigningKey("different-disposable-key")
|
||||
}
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.want, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now))
|
||||
})
|
||||
}
|
||||
require.False(t, isExpiredVolumeJwtAt(key, "malformed", "1,0000000100001234", now))
|
||||
require.False(t, isExpiredVolumeJwtAt(nil, "malformed", "1,0000000100001234", now))
|
||||
// Bad temporal field types must fail parsing instead of producing a marker.
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"fid": "1,0000000100001234", "exp": now.Add(-time.Minute).Unix(), "nbf": "not-a-date"}).SignedString([]byte(key))
|
||||
require.NoError(t, err)
|
||||
require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now))
|
||||
}
|
||||
|
||||
func TestVolumeJwtExpiryAlgorithmBoundary(t *testing.T) {
|
||||
now := time.Unix(1900000000, 0)
|
||||
key := SigningKey("disposable-algorithm-test")
|
||||
claims := SeaweedFileIdClaims{Fid: "1,0000000100001234", RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(now.Add(-time.Second))}}
|
||||
// Existing DecodeJwt permits the HMAC family, not just HS256.
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS512, claims).SignedString([]byte(key))
|
||||
require.NoError(t, err)
|
||||
require.True(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), claims.Fid, now))
|
||||
unsigned, err := jwt.NewWithClaims(jwt.SigningMethodNone, claims).SignedString(jwt.UnsafeAllowNoneSignatureType)
|
||||
require.NoError(t, err)
|
||||
require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(unsigned), claims.Fid, now))
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type expiryAssignRPC struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
assign func() (*filer_pb.AssignVolumeResponse, error)
|
||||
}
|
||||
|
||||
func (c expiryAssignRPC) AssignVolume(_ context.Context, _ *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) {
|
||||
return c.assign()
|
||||
}
|
||||
|
||||
type expiryFilerClient struct{ rpc expiryAssignRPC }
|
||||
|
||||
func (c expiryFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return fn(c.rpc)
|
||||
}
|
||||
func (c expiryFilerClient) AdjustedUrl(l *filer_pb.Location) string { return l.Url }
|
||||
func (c expiryFilerClient) GetDataCenter() string { return "" }
|
||||
|
||||
type expiryCountingBody struct {
|
||||
io.ReadCloser
|
||||
bytesRead int
|
||||
}
|
||||
|
||||
func (b *expiryCountingBody) Read(p []byte) (int, error) {
|
||||
n, e := b.ReadCloser.Read(p)
|
||||
b.bytesRead += n
|
||||
return n, e
|
||||
}
|
||||
|
||||
func expiryVolume(t *testing.T, key security.SigningKey) *VolumeServer {
|
||||
t.Helper()
|
||||
s := newTierTestStore(t, t.TempDir())
|
||||
require.NoError(t, s.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0))
|
||||
return &VolumeServer{store: s, guard: security.NewGuard(nil, string(key), 10, "", 0), fileSizeLimitBytes: 4 << 20}
|
||||
}
|
||||
|
||||
func expirySignedToken(t *testing.T, key security.SigningKey, fid string, expires time.Time) string {
|
||||
t.Helper()
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(expires)}}).SignedString([]byte(key))
|
||||
require.NoError(t, err)
|
||||
return token
|
||||
}
|
||||
|
||||
// Assignment returns a genuinely valid ten-second JWT. The HTTP server delays
|
||||
// only that assignment until its exact expiry boundary before running the real
|
||||
// PostHandler. This bounded wait reproduces transit/preauthorization delay; it
|
||||
// makes no CPU-throughput or shared-clock-injection assumption.
|
||||
func TestExpiredVolumeJwtReassignmentHTTP(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
forever, cipher bool
|
||||
}{{"bounded", false, false}, {"bounded_cipher", false, true}, {"retry_forever", true, false}, {"retry_forever_cipher", true, true}} {
|
||||
name, forever := tc.name, tc.forever
|
||||
t.Run(name, func(t *testing.T) {
|
||||
key := security.SigningKey("disposable-volume-expiry-test")
|
||||
vs := expiryVolume(t, key)
|
||||
plain := bytes.Repeat([]byte("compressible lifecycle payload"), 4096)
|
||||
sum := md5.Sum(plain)
|
||||
var mu sync.Mutex
|
||||
var ids []string
|
||||
var statuses []int
|
||||
var outcomes []string
|
||||
var bodyReads []int
|
||||
var fsyncs []string
|
||||
var assigned int
|
||||
var firstExpires time.Time
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
expires := firstExpires
|
||||
mu.Unlock()
|
||||
if r.URL.Path == "/1,0000000100001234" {
|
||||
if delay := time.Until(expires.Add(100 * time.Millisecond)); delay > 0 {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
counted := &expiryCountingBody{ReadCloser: r.Body}
|
||||
r.Body = counted
|
||||
rec := httptest.NewRecorder()
|
||||
vs.PostHandler(rec, r)
|
||||
mu.Lock()
|
||||
ids = append(ids, r.URL.Path)
|
||||
statuses = append(statuses, rec.Code)
|
||||
outcomes = append(outcomes, rec.Header().Get("X-Seaweed-Write-Outcome"))
|
||||
bodyReads = append(bodyReads, counted.bytesRead)
|
||||
fsyncs = append(fsyncs, r.URL.Query().Get("fsync"))
|
||||
mu.Unlock()
|
||||
for k, v := range rec.Header() {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(rec.Code)
|
||||
_, _ = w.Write(rec.Body.Bytes())
|
||||
}))
|
||||
defer server.Close()
|
||||
c := expiryFilerClient{rpc: expiryAssignRPC{assign: func() (*filer_pb.AssignVolumeResponse, error) {
|
||||
assigned++
|
||||
fid := "1,0000000100001234"
|
||||
expiry := time.Now().Add(10 * time.Second)
|
||||
if assigned == 1 {
|
||||
mu.Lock()
|
||||
firstExpires = expiry
|
||||
mu.Unlock()
|
||||
}
|
||||
if assigned > 1 {
|
||||
fid = "1,0000000200001234"
|
||||
expiry = time.Now().Add(time.Hour)
|
||||
}
|
||||
token := expirySignedToken(t, key, fid, expiry)
|
||||
parsed, decodeErr := security.DecodeJwt(key, security.EncodedJwt(token), &security.SeaweedFileIdClaims{})
|
||||
require.NoError(t, decodeErr)
|
||||
require.True(t, parsed.Valid)
|
||||
return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: token, Fsync: true, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}}, nil
|
||||
}}}
|
||||
httpClient := server.Client()
|
||||
httpClient.Timeout = 25 * time.Second
|
||||
uploader := operation.NewUploaderWithHttpClient(httpClient)
|
||||
fid, result, err, returned := uploader.UploadWithRetry(c, &filer_pb.AssignVolumeRequest{Path: "/own/long.bin"}, &operation.UploadOption{Cipher: tc.cipher, Filename: "long.bin", MimeType: "application/octet-stream", WantMd5: true, RetryForever: forever}, bytes.NewReader(plain))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "1,0000000200001234", fid)
|
||||
require.Equal(t, 2, assigned)
|
||||
require.Equal(t, plain, returned)
|
||||
require.Equal(t, uint32(len(plain)), result.Size)
|
||||
require.Equal(t, base64.StdEncoding.EncodeToString(sum[:]), result.ContentMd5)
|
||||
mu.Lock()
|
||||
gotIds := append([]string(nil), ids...)
|
||||
gotStatuses := append([]int(nil), statuses...)
|
||||
gotFsyncs := append([]string(nil), fsyncs...)
|
||||
gotOutcomes := append([]string(nil), outcomes...)
|
||||
gotBodyReads := append([]int(nil), bodyReads...)
|
||||
mu.Unlock()
|
||||
require.Equal(t, []string{"/1,0000000100001234", "/1,0000000200001234"}, gotIds)
|
||||
require.Equal(t, []int{http.StatusUnauthorized, http.StatusCreated}, gotStatuses)
|
||||
require.Equal(t, []string{"true", "true"}, gotFsyncs)
|
||||
require.Equal(t, []string{"expired-jwt-before-append", ""}, gotOutcomes)
|
||||
require.Equal(t, 0, gotBodyReads[0])
|
||||
require.Positive(t, gotBodyReads[1])
|
||||
require.Equal(t, uint32(1), result.Gzip)
|
||||
if tc.cipher {
|
||||
require.NotEmpty(t, result.CipherKey)
|
||||
} else {
|
||||
require.Empty(t, result.CipherKey)
|
||||
}
|
||||
_, err = vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil)
|
||||
require.Error(t, err)
|
||||
stored := &needle.Needle{Id: 2, Cookie: 0x1234}
|
||||
_, err = vs.store.ReadVolumeNeedle(1, stored, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, stored.Data)
|
||||
recovered := stored.Data
|
||||
if tc.cipher {
|
||||
recovered, err = util.Decrypt(recovered, util.CipherKey(result.CipherKey))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
if result.Gzip > 0 {
|
||||
recovered, err = util.DecompressData(recovered)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, plain, recovered)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeJwtExpiryRefusalBoundary(t *testing.T) {
|
||||
key := security.SigningKey("disposable-volume-boundary")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
expired, nbf, iat, badKey, badFid, missing, urlencoded, suffix, noKey bool
|
||||
want int
|
||||
marker bool
|
||||
}{
|
||||
{name: "expired", expired: true, want: 401, marker: true},
|
||||
{name: "expired_suffix", expired: true, suffix: true, want: 401, marker: true},
|
||||
{name: "expired_future_nbf", expired: true, nbf: true, want: 401},
|
||||
{name: "expired_future_iat", expired: true, iat: true, want: 401},
|
||||
{name: "expired_wrong_signature", expired: true, badKey: true, want: 401},
|
||||
{name: "expired_wrong_fid", expired: true, badFid: true, want: 401},
|
||||
{name: "missing", missing: true, want: 401},
|
||||
{name: "nonmultipart", expired: true, urlencoded: true, want: 401},
|
||||
{name: "valid", want: 201},
|
||||
{name: "no_key", missing: true, noKey: true, want: 201},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
vs := expiryVolume(t, key)
|
||||
if tc.noKey {
|
||||
vs.guard = security.NewGuard(nil, "", 0, "", 0)
|
||||
}
|
||||
plain := []byte("boundary content")
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
part, e := mw.CreateFormFile("file", "sample.bin")
|
||||
require.NoError(t, e)
|
||||
_, e = part.Write(plain)
|
||||
require.NoError(t, e)
|
||||
require.NoError(t, mw.Close())
|
||||
contentType := mw.FormDataContentType()
|
||||
if tc.urlencoded {
|
||||
body.Reset()
|
||||
body.WriteString("sample=value")
|
||||
contentType = "application/x-www-form-urlencoded"
|
||||
}
|
||||
counted := &expiryCountingBody{ReadCloser: io.NopCloser(bytes.NewReader(body.Bytes()))}
|
||||
path := "/1,0000000100001234"
|
||||
if tc.suffix {
|
||||
path += "_2"
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, path, counted)
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
// A request-supplied marker must never be echoed as response proof.
|
||||
req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append")
|
||||
if !tc.missing {
|
||||
fid := "1,0000000100001234"
|
||||
if tc.badFid {
|
||||
fid = "1,0000000200001234"
|
||||
}
|
||||
exp := time.Now().Add(time.Hour)
|
||||
if tc.expired {
|
||||
exp = time.Now().Add(-time.Hour)
|
||||
}
|
||||
claims := security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}}
|
||||
if tc.nbf {
|
||||
claims.NotBefore = jwt.NewNumericDate(time.Now().Add(time.Hour))
|
||||
}
|
||||
if tc.iat {
|
||||
claims.IssuedAt = jwt.NewNumericDate(time.Now().Add(time.Hour))
|
||||
}
|
||||
signingKey := key
|
||||
if tc.badKey {
|
||||
signingKey = security.SigningKey("different-disposable-key")
|
||||
}
|
||||
token, e := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey))
|
||||
require.NoError(t, e)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
vs.PostHandler(rec, req)
|
||||
require.Equal(t, tc.want, rec.Code)
|
||||
want := ""
|
||||
if tc.marker {
|
||||
want = "expired-jwt-before-append"
|
||||
}
|
||||
require.Equal(t, want, rec.Header().Get("X-Seaweed-Write-Outcome"))
|
||||
n := &needle.Needle{Id: 1, Cookie: 0x1234}
|
||||
_, readErr := vs.store.ReadVolumeNeedle(1, n, nil, nil)
|
||||
if tc.want == 401 {
|
||||
require.Error(t, readErr)
|
||||
if !tc.urlencoded {
|
||||
require.Zero(t, counted.bytesRead)
|
||||
}
|
||||
} else {
|
||||
require.NoError(t, readErr)
|
||||
require.Equal(t, plain, n.Data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeJwtExpiryProxyForwarding(t *testing.T) {
|
||||
volumeKey := security.SigningKey("disposable-proxy-volume-key")
|
||||
filerKey := security.SigningKey("disposable-proxy-filer-key")
|
||||
vs := expiryVolume(t, volumeKey)
|
||||
var hits atomic.Int32
|
||||
volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if hits.Add(1) == 1 {
|
||||
claims := &security.SeaweedFileIdClaims{}
|
||||
parsed, err := security.DecodeJwt(volumeKey, security.GetJwt(r), claims)
|
||||
if err != nil || !parsed.Valid || claims.ExpiresAt == nil {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond)))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
vs.PostHandler(w, r)
|
||||
}))
|
||||
defer volume.Close()
|
||||
fs := &FilerServer{filerGuard: security.NewGuard(nil, string(filerKey), 3600, "", 0), volumeGuard: security.NewGuard(nil, string(volumeKey), 10, "", 0)}
|
||||
fid := "1,0000000100001234"
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Exercise the same real caller verifier as filerHandler, then its actual
|
||||
// forwarding function with an already-resolved synthetic target.
|
||||
if _, ok := fs.authenticateFilerJwt(r, true); !ok {
|
||||
w.WriteHeader(401)
|
||||
return
|
||||
}
|
||||
fs.proxyToVolumeServerURL(w, r, fid, volume.URL+"/"+fid)
|
||||
}))
|
||||
defer proxy.Close()
|
||||
caller := string(security.GenJwtForFilerServer(filerKey, 3600))
|
||||
for i, want := range []int{401, 401, 201} {
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
part, err := mw.CreateFormFile("file", "proxy.bin")
|
||||
require.NoError(t, err)
|
||||
_, err = part.Write([]byte("proxy clear data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, mw.Close())
|
||||
req, err := http.NewRequest(http.MethodPost, proxy.URL+"/?proxyChunkId="+fid, bytes.NewReader(body.Bytes()))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append")
|
||||
token := caller
|
||||
if i == 1 {
|
||||
token = "invalid-disposable-caller"
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
client := proxy.Client()
|
||||
client.Timeout = 25 * time.Second
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(io.Discard, resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, want, resp.StatusCode)
|
||||
marker := ""
|
||||
if i == 0 {
|
||||
marker = "expired-jwt-before-append"
|
||||
}
|
||||
require.Equal(t, marker, resp.Header.Get("X-Seaweed-Write-Outcome"))
|
||||
if i == 1 {
|
||||
require.Equal(t, int32(1), hits.Load())
|
||||
}
|
||||
}
|
||||
require.Equal(t, int32(2), hits.Load())
|
||||
}
|
||||
|
||||
func TestVolumeJwtExpiryFanoutRollback(t *testing.T) {
|
||||
key := security.SigningKey("disposable-fanout-expiry-key")
|
||||
primary, replica := expiryVolume(t, key), expiryVolume(t, key)
|
||||
firstCommitted := make(chan struct{})
|
||||
var committed sync.Once
|
||||
var mu sync.Mutex
|
||||
var firstReplicaStatuses []int
|
||||
var firstReplicaMarkers []string
|
||||
a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
primary.DeleteHandler(w, r)
|
||||
return
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
primary.PostHandler(rec, r)
|
||||
if r.URL.Path == "/1,0000000100001234" && rec.Code == 201 {
|
||||
committed.Do(func() { close(firstCommitted) })
|
||||
}
|
||||
for k, v := range rec.Header() {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(rec.Code)
|
||||
_, _ = w.Write(rec.Body.Bytes())
|
||||
}))
|
||||
defer a.Close()
|
||||
b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
replica.DeleteHandler(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/1,0000000100001234" {
|
||||
select {
|
||||
case <-firstCommitted:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
claims := &security.SeaweedFileIdClaims{}
|
||||
parsed, err := security.DecodeJwt(key, security.GetJwt(r), claims)
|
||||
if err != nil || !parsed.Valid || claims.ExpiresAt == nil {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond)))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
replica.PostHandler(rec, r)
|
||||
if r.URL.Path == "/1,0000000100001234" {
|
||||
mu.Lock()
|
||||
firstReplicaStatuses = append(firstReplicaStatuses, rec.Code)
|
||||
firstReplicaMarkers = append(firstReplicaMarkers, rec.Header().Get("X-Seaweed-Write-Outcome"))
|
||||
mu.Unlock()
|
||||
}
|
||||
for k, v := range rec.Header() {
|
||||
w.Header()[k] = v
|
||||
}
|
||||
w.WriteHeader(rec.Code)
|
||||
_, _ = w.Write(rec.Body.Bytes())
|
||||
}))
|
||||
defer b.Close()
|
||||
var assignments int
|
||||
plain := bytes.Repeat([]byte("fanout content"), 4096)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
|
||||
defer cancel()
|
||||
result, err := operation.UploadReaderInChunks(ctx, bytes.NewReader(plain), &operation.ChunkedUploadOption{
|
||||
ChunkSize: 1 << 20, MimeType: "application/octet-stream", AssignFunc: func(_ context.Context, _ int, _ uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
|
||||
assignments++
|
||||
fid := "1,0000000100001234"
|
||||
if assignments > 1 {
|
||||
fid = "1,0000000200001234"
|
||||
}
|
||||
return nil, &operation.AssignResult{Fid: fid, Url: strings.TrimPrefix(a.URL, "http://"), Auth: security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(10*time.Second))), Replicas: []operation.Location{{Url: strings.TrimPrefix(b.URL, "http://")}}, Fsync: true}, nil
|
||||
}, LookupJwtFunc: func(_ context.Context, fid string) security.EncodedJwt {
|
||||
return security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(time.Hour)))
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, assignments)
|
||||
mu.Lock()
|
||||
gotStatuses := append([]int(nil), firstReplicaStatuses...)
|
||||
gotMarkers := append([]string(nil), firstReplicaMarkers...)
|
||||
mu.Unlock()
|
||||
require.Equal(t, []int{401}, gotStatuses)
|
||||
require.Equal(t, []string{"expired-jwt-before-append"}, gotMarkers)
|
||||
require.Len(t, result.FileChunks, 1)
|
||||
require.Equal(t, "1,0000000200001234", result.FileChunks[0].FileId)
|
||||
for _, vs := range []*VolumeServer{primary, replica} {
|
||||
_, err := vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil)
|
||||
require.Error(t, err)
|
||||
n := &needle.Needle{Id: 2, Cookie: 0x1234}
|
||||
_, err = vs.store.ReadVolumeNeedle(1, n, nil, nil)
|
||||
require.NoError(t, err)
|
||||
data := n.Data
|
||||
if result.FileChunks[0].IsCompressed {
|
||||
data, err = util.DecompressData(data)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Equal(t, plain, data)
|
||||
}
|
||||
}
|
||||
@@ -330,19 +330,23 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) maybeCheckJwtAuthorization(r *http.Request, vid, fid string, isWrite bool) bool {
|
||||
|
||||
var signingKey security.SigningKey
|
||||
|
||||
if isWrite {
|
||||
signingKey = vs.guard.SigningKey()
|
||||
if len(signingKey) == 0 {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
signingKey = vs.guard.ReadSigningKey()
|
||||
if len(signingKey) == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return vs.checkJwtAuthorizationWithKey(r, vid, fid, signingKey)
|
||||
}
|
||||
|
||||
// checkJwtAuthorizationWithKey authorizes against a caller-snapshotted signing
|
||||
// key, so a write handler can classify an expired-JWT refusal against the exact
|
||||
// key it authorized under rather than re-reading one that may have rotated
|
||||
// between the two reads.
|
||||
func (vs *VolumeServer) checkJwtAuthorizationWithKey(r *http.Request, vid, fid string, signingKey security.SigningKey) bool {
|
||||
|
||||
if len(signingKey) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
tokenStr := security.GetJwt(r)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -14,11 +15,13 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
var jpegStart = []byte{0xFF, 0xD8, 0xFF}
|
||||
@@ -52,7 +55,22 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
|
||||
// Snapshot the write signing key once, so the authorization check and the
|
||||
// expired-JWT classification below read the same key even if it rotates.
|
||||
writeSigningKey := vs.guard.SigningKey()
|
||||
if !vs.checkJwtAuthorizationWithKey(r, vid, fid, writeSigningKey) {
|
||||
// ParseForm does not read multipart bodies. Preserve its existing placement
|
||||
// and status precedence for other content types, which get no retry marker.
|
||||
contentType, _, typeErr := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if typeErr == nil && contentType == "multipart/form-data" {
|
||||
baseFid := fid
|
||||
if sep := strings.LastIndex(baseFid, "_"); sep > 0 {
|
||||
baseFid = baseFid[:sep]
|
||||
}
|
||||
if security.IsExpiredVolumeJwt(writeSigningKey, security.GetJwt(r), vid+","+baseFid) {
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
|
||||
}
|
||||
}
|
||||
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
|
||||
return
|
||||
}
|
||||
@@ -77,7 +95,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
status = http.StatusConflict
|
||||
}
|
||||
if errors.Is(writeError, topology.ErrorWriteRefused) {
|
||||
w.Header().Set("X-Seaweed-Write-Outcome", "refused-before-append")
|
||||
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeRefusedBeforeAppend)
|
||||
}
|
||||
writeJsonError(w, r, status, writeError)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package http
|
||||
|
||||
// Write outcomes describe a refusal of this HTTP attempt before append, not the
|
||||
// absence of a stored needle from an earlier attempt with a lost response.
|
||||
const (
|
||||
WriteOutcomeHeader = "X-Seaweed-Write-Outcome"
|
||||
WriteOutcomeRefusedBeforeAppend = "refused-before-append"
|
||||
WriteOutcomeExpiredJwtBeforeAppend = "expired-jwt-before-append"
|
||||
)
|
||||
Reference in new issue
Block a user