operation: reassign an upload after a proven-expired volume JWT (#12729)

* operation: reassign an upload after a proven-expired volume JWT

* seaweed-volume: mark an expired write JWT refused before append

Mirror the Go volume server: a multipart POST whose write JWT is signed
with the current key, names this fid and fails only on exp gets
X-Seaweed-Write-Outcome: expired-jwt-before-append on its 401, so the
uploader reassigns instead of failing. Both checks use one guard
snapshot, so a key rotation between them cannot drop the marker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: keep same-FID retry for filer-proxy expiries, snapshot the write signing key, and make the RetryForever budget test fail fast

* operation: re-mint the filer credential before a proxy same-FID expiry retry

* operation: make a proxy marked-expiry terminal in the outer retry too, not only the inner same-FID loop

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-18 11:02:52 -08:00
1 parent ecbf7af1ca
commit ebc77400ca
11 files changed
+1215 -29

No files matched your search

+80
View File
@@ -106,6 +106,39 @@ pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims,
Ok(data.claims)
}
/// Reports a signed, fid-bound write token whose only failure is expiry. It
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
pub fn is_expired_volume_jwt(signing_key: &SigningKey, token: &str, expected_fid: &str) -> bool {
match decode_jwt(signing_key, token) {
Err(JwtError::Jwt(e))
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
_ => return false,
}
#[derive(Deserialize)]
struct ExpiredClaims {
fid: Option<String>,
iat: Option<u64>,
}
let mut validation = Validation::new(Algorithm::HS256);
validation.required_spec_claims.clear();
validation.validate_exp = false;
validation.validate_nbf = true;
validation.leeway = 0;
let Ok(data) = decode::<ExpiredClaims>(
token,
&DecodingKey::from_secret(&signing_key.0),
&validation,
) else {
return false;
};
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
}
// ============================================================================
// Guard
// ============================================================================
@@ -410,6 +443,53 @@ mod tests {
assert!(decode_jwt(&key2, &token).is_err());
}
#[test]
fn test_is_expired_volume_jwt_classifies_expiry_only() {
let key = SigningKey::from_string("write-key");
let fid = "3,01637037d6";
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let sign = |claims: serde_json::Value, key: &SigningKey| {
encode(
&Header::new(Algorithm::HS256),
&claims,
&EncodingKey::from_secret(&key.0),
)
.unwrap()
};
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
assert!(is_expired_volume_jwt(&key, &expired, fid));
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7"));
assert!(!is_expired_volume_jwt(
&SigningKey::from_string("other-key"),
&expired,
fid
));
assert!(!is_expired_volume_jwt(&SigningKey(vec![]), &expired, fid));
let immature = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &immature, fid));
let future_issued = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &future_issued, fid));
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
assert!(!is_expired_volume_jwt(&key, &live, fid));
let unsigned = format!(
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
base64::Engine::encode(
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
)
);
assert!(!is_expired_volume_jwt(&key, &unsigned, fid));
}
#[test]
fn test_guard_empty_whitelist() {
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
+111 -8
View File
@@ -483,6 +483,18 @@ fn extract_file_id(path: &str) -> String {
format!("{},{}", parsed.vid, fid)
}
fn is_multipart_form_data(headers: &HeaderMap) -> bool {
headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.split(';').next())
.is_some_and(|media_type| {
media_type
.trim()
.eq_ignore_ascii_case("multipart/form-data")
})
}
fn streaming_chunk_size(read_buffer_size_bytes: usize, data_size: usize) -> usize {
std::cmp::min(
read_buffer_size_bytes.max(DEFAULT_STREAMING_CHUNK_SIZE),
@@ -2777,14 +2789,28 @@ pub async fn post_handler(
// JWT check for writes
let file_id = extract_file_id(&path);
let token = extract_jwt(&headers, request.uri());
if state
.guard
.read()
.unwrap()
.check_jwt_for_file(token.as_deref(), &file_id, true)
.is_err()
{
return json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
let expired_jwt = {
let guard = state.guard.read().unwrap();
match guard.check_jwt_for_file(token.as_deref(), &file_id, true) {
Ok(()) => None,
Err(_) => Some(
is_multipart_form_data(&headers)
&& token.as_deref().is_some_and(|token| {
crate::security::is_expired_volume_jwt(&guard.signing_key, token, &file_id)
}),
),
}
};
if let Some(expired_jwt) = expired_jwt {
let mut response =
json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
if expired_jwt {
response.headers_mut().insert(
"x-seaweed-write-outcome",
header::HeaderValue::from_static("expired-jwt-before-append"),
);
}
return response;
}
// Upload throttling: check inflight bytes against limit
@@ -5465,6 +5491,83 @@ mod tests {
assert_eq!(extract_jwt(&headers, &uri), None);
}
#[tokio::test]
async fn http_expired_write_jwt_is_marked_before_append() {
use crate::security::SigningKey;
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
use tower::ServiceExt;
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let key = SigningKey(b"write-secret".to_vec());
state.guard.write().unwrap().signing_key = key.clone();
let volume = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
let before = volume.dat_file_size().unwrap();
let fid = format!("1,{:x}{TEST_COOKIE:08x}", 0x6e7a_0c09u64);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let sign = |key: &[u8]| {
encode(
&Header::new(Algorithm::HS256),
&serde_json::json!({"fid": fid, "exp": now - 10}),
&EncodingKey::from_secret(key),
)
.unwrap()
};
let boundary = "expiredjwtboundary";
let body = format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a\"\r\n\r\nexpired-bytes\r\n--{boundary}--\r\n"
);
for (token, content_type, marked) in [
(
sign(&key.0),
format!("multipart/form-data; boundary={boundary}"),
true,
),
(
sign(b"other-secret"),
format!("multipart/form-data; boundary={boundary}"),
false,
),
(sign(&key.0), "application/octet-stream".to_string(), false),
] {
let response = super::super::volume_server::build_admin_router(state.clone())
.oneshot(
Request::builder()
.method(Method::POST)
.uri(format!("/{fid}"))
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(header::CONTENT_TYPE, &content_type)
.body(Body::from(body.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::UNAUTHORIZED,
"{content_type}"
);
assert_eq!(
response
.headers()
.get("x-seaweed-write-outcome")
.map(|value| value.to_str().unwrap()),
marked.then_some("expired-jwt-before-append"),
"{content_type}"
);
}
assert_eq!(volume.dat_file_size().unwrap(), before);
}
#[test]
fn test_handle_range_single() {
let data = b"hello world";
+49 -10
View File
@@ -131,6 +131,7 @@ var (
type uploadStatusError struct {
StatusCode int
refusedBeforeAppend bool
expiredBeforeAppend bool
stalled bool // the stall watchdog cancelled this attempt's request
err error
}
@@ -149,7 +150,7 @@ func ShouldReassignUpload(err error) bool {
}
var se *uploadStatusError
if errors.As(err, &se) {
return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend)
return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend) || (se.StatusCode == http.StatusUnauthorized && se.expiredBeforeAppend)
}
return strings.Contains(err.Error(), "transport")
}
@@ -243,17 +244,37 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho
return err
}
// A credential can expire while the first HTTP request is queued. Bound
// expiry-specific reassignment even when unrelated transient retries are
// intentionally unbounded. The third marked refusal is final.
expiryRefusals := 0
shouldRetry := func(err error) bool {
var statusErr *uploadStatusError
if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend {
// A proxy upload already exhausted its bounded same-FID retries in
// retriedUploadData (with a fresh filer credential each time); the
// filer owns the FID, so a reassignment here is wrong, and counting
// exhausted groups would also inflate the refusal budget. Make it
// terminal; only direct uploads reassign.
if util_http.IsProxyChunkUrl(uploadOption.UploadUrl) {
return false
}
expiryRefusals++
return expiryRefusals < 3
}
return ShouldReassignUpload(err)
}
if uploadOption.RetryForever {
util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
glog.V(0).Infof("upload content: %v", err)
var statusErr *uploadStatusError
if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusConflict {
return ShouldReassignUpload(err)
if errors.As(err, &statusErr) && (statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend)) {
return shouldRetry(err)
}
return true
})
} else {
err = util.RetryOnError("uploadWithRetry", ShouldReassignUpload, doUploadFunc)
err = util.RetryOnError("uploadWithRetry", shouldRetry, doUploadFunc)
}
return
@@ -380,9 +401,26 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *prepa
}
var statusErr *uploadStatusError
if errors.As(err, &statusErr) {
if statusErr.StatusCode == http.StatusConflict {
// A marked-expiry 401 is terminal here only for a direct volume
// upload: its assignment JWT is spent, so recovery needs a fresh
// assignment (uploadWithRetryData). A filer-proxy upload instead
// mints a fresh volume JWT on each forwarded request, so a bounded
// same-FID retry is enough and reassignment is both unnecessary and
// wrong (the proxy owns the FID). Keep that same-FID retry for it.
if statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && !util_http.IsProxyChunkUrl(option.UploadUrl)) {
return nil, err
}
if statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && util_http.IsProxyChunkUrl(option.UploadUrl) {
// The caller's filer credential was minted once and may have
// expired alongside the volume JWT (both default to 10s), so
// re-mint it before replaying the same FID; otherwise the filer
// rejects the retry with an unmarked 401 and recovery stops.
if option.FilerJwt != nil {
option.Jwt = option.FilerJwt(true)
} else {
option.Jwt = security.EncodedJwt(util_http.JwtForFilerServer(true))
}
}
if statusErr.stalled {
option.stallWatchdogDisarmed = true
}
@@ -720,26 +758,27 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
respStallTimer := time.AfterFunc(uploadStallMinWait, stallFired)
defer respStallTimer.Stop()
}
refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get("X-Seaweed-Write-Outcome") == "refused-before-append"
refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeRefusedBeforeAppend
expiredBeforeAppend := resp.StatusCode == http.StatusUnauthorized && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeExpiredJwtBeforeAppend
resp_body, ra_err := io.ReadAll(resp.Body)
uploadDone.Store(true)
if ra_err != nil {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)}
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)}
}
unmarshal_err := json.Unmarshal(resp_body, &ret)
if unmarshal_err != nil {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body))
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)}
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)}
}
if resp.StatusCode == http.StatusConflict && ret.Error == "" {
if (resp.StatusCode == http.StatusConflict || expiredBeforeAppend) && ret.Error == "" {
ret.Error = http.StatusText(resp.StatusCode)
}
if ret.Error != "" {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)}
return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)}
}
if bps := uploadObservedBps(bodySize, movedIn); bps > 0 {
uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps})
@@ -0,0 +1,357 @@
package operation
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/security"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"github.com/stretchr/testify/require"
)
func TestUploadExpiryResponseContract(t *testing.T) {
for _, tc := range []struct {
name string
status int
marker, body string
want bool
calls int
}{
{"marked", 401, "expired-jwt-before-append", `{"error":"wrong jwt"}`, true, 1},
{"marked_empty_error", 401, "expired-jwt-before-append", `{}`, true, 1},
{"marked_bad_json", 401, "expired-jwt-before-append", `{`, true, 1},
{"unmarked", 401, "", `{"error":"wrong jwt"}`, false, 3},
{"wrong_value", 401, "expired-jwt-before-append-extra", `{"error":"wrong jwt"}`, false, 3},
{"wrong_status", 403, "expired-jwt-before-append", `{"error":"denied"}`, false, 3},
{"existing_409", 409, "refused-before-append", `{"error":"full"}`, true, 1},
} {
t.Run(tc.name, func(t *testing.T) {
var mu sync.Mutex
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
mu.Unlock()
w.Header().Set(util_http.WriteOutcomeHeader, tc.marker)
w.WriteHeader(tc.status)
_, _ = io.WriteString(w, tc.body)
}))
defer srv.Close()
u := NewUploaderWithHttpClient(srv.Client())
_, err := u.UploadData(context.Background(), []byte("direct preassigned data"), &UploadOption{UploadUrl: srv.URL, Jwt: "disposable-preassigned"})
require.Error(t, err)
require.Equal(t, tc.want, ShouldReassignUpload(err))
mu.Lock()
actual := calls
mu.Unlock()
require.Equal(t, tc.calls, actual)
})
}
}
func TestUploadExpiryBudgetAndAssignmentFailure(t *testing.T) {
for _, tc := range []struct {
name string
forever, assignmentFailure bool
}{{"bounded", false, false}, {"forever", true, false}, {"assignment_failure", false, true}} {
t.Run(tc.name, func(t *testing.T) {
var mu sync.Mutex
calls := 0
var payloads [][]byte
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mr, err := r.MultipartReader()
if err != nil {
w.WriteHeader(400)
return
}
part, err := mr.NextPart()
if err != nil {
w.WriteHeader(400)
return
}
data, err := io.ReadAll(part)
if err != nil {
w.WriteHeader(400)
return
}
mu.Lock()
calls++
n := calls
payloads = append(payloads, data)
mu.Unlock()
// Terminal fallback: if the expiry budget regressed and did not
// stop at 3, end the run with an unmarked 409 so this test fails
// here instead of looping to the package timeout under RetryForever.
if n > 3 {
w.WriteHeader(409)
_, _ = io.WriteString(w, `{"error":"full"}`)
return
}
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
}))
defer srv.Close()
u := NewUploaderWithHttpClient(srv.Client())
assigns := 0
sentinel := errors.New("synthetic assignment failure")
_, _, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
assigns++
if tc.assignmentFailure && assigns == 2 {
return "", "", "", false, sentinel
}
return fmt.Sprintf("1,%08x00001234", assigns), strings.TrimPrefix(srv.URL, "http://"), "disposable-per-assignment", false, nil
}, &UploadOption{Cipher: true, MimeType: "application/octet-stream", RetryForever: tc.forever}, bytes.Repeat([]byte("compressible data"), 1024))
require.Error(t, err)
mu.Lock()
actual := calls
got := append([][]byte(nil), payloads...)
mu.Unlock()
if tc.assignmentFailure {
require.ErrorIs(t, err, sentinel)
require.Equal(t, 2, assigns)
require.Equal(t, 1, actual)
} else {
require.Equal(t, 3, assigns)
require.Equal(t, 3, actual)
// The run must end on the 3rd marked 401 (the budget), not the
// 409 terminal fallback above.
var finalErr *uploadStatusError
require.ErrorAs(t, err, &finalErr)
require.Equal(t, http.StatusUnauthorized, finalErr.StatusCode)
require.True(t, finalErr.expiredBeforeAppend)
}
require.NotEmpty(t, got)
for _, data := range got {
require.Equal(t, got[0], data)
}
})
}
}
// A filer-proxy upload must retry a marked-expiry 401 on the SAME FID -- the
// proxy mints a fresh volume JWT per forwarded request, so a same-FID replay
// recovers without a reassignment it does not own.
func TestUploadExpiryProxyRetriesSameFid(t *testing.T) {
const fid = "3,01637037d6"
var mu sync.Mutex
calls := 0
var seenFids []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
n := calls
seenFids = append(seenFids, r.URL.Query().Get(util_http.ProxyChunkIdParam))
mu.Unlock()
if n == 1 {
// The proxy-minted volume JWT expired in transit on the first try.
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
return
}
_, _ = io.WriteString(w, `{"size":5}`)
}))
defer srv.Close()
proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid
result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: proxyURL})
require.NoError(t, err)
require.Equal(t, uint32(5), result.Size)
mu.Lock()
defer mu.Unlock()
require.Equal(t, 2, calls)
// Both forwarded requests used the same FID: a proxy upload never reassigns.
require.Equal(t, []string{fid, fid}, seenFids)
}
// With filer signing enabled, the caller's filer credential can expire together
// with the volume JWT. The proxy same-FID retry must carry a freshly minted
// filer credential, not replay the stale one (which the filer would reject with
// an unmarked 401 that stops recovery).
func TestUploadExpiryProxyRefreshesFilerJwtOnRetry(t *testing.T) {
const fid = "3,01637037d6"
var mu sync.Mutex
calls := 0
var auths []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
n := calls
auths = append(auths, r.Header.Get("Authorization"))
mu.Unlock()
if n == 1 {
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
return
}
_, _ = io.WriteString(w, `{"size":5}`)
}))
defer srv.Close()
proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid
minted := 0
option := &UploadOption{UploadUrl: proxyURL, Jwt: "stale-filer-jwt", FilerJwt: func(isWrite bool) security.EncodedJwt {
minted++
return security.EncodedJwt(fmt.Sprintf("fresh-filer-jwt-%d", minted))
}}
result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), option)
require.NoError(t, err)
require.Equal(t, uint32(5), result.Size)
mu.Lock()
defer mu.Unlock()
require.Equal(t, 2, calls)
require.Equal(t, 1, minted)
// The first request carried the stale credential; the retry re-minted one.
require.Equal(t, "Bearer stale-filer-jwt", auths[0])
require.Equal(t, "Bearer fresh-filer-jwt-1", auths[1])
}
// A proxy upload driven through uploadWithRetryData must be terminal after its
// bounded same-FID attempts: the outer loop must not reassign a FID the proxy
// owns, nor count exhausted groups toward the expiry budget.
func TestUploadExpiryProxyDoesNotReassignThroughOuterRetry(t *testing.T) {
var mu sync.Mutex
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
mu.Unlock()
// The proxy keeps reporting the marked expiry (never recovers).
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
}))
defer srv.Close()
host := strings.TrimPrefix(srv.URL, "http://")
assigns := 0
option := &UploadOption{
FilerJwt: func(isWrite bool) security.EncodedJwt { return "fresh-filer-jwt" },
GenUploadUrl: func(host, fileId string) string {
return "http://" + host + "/?" + util_http.ProxyChunkIdParam + "=" + fileId
},
}
_, _, err := NewUploaderWithHttpClient(srv.Client()).uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
assigns++
return "3,01637037d6", host, "vol-jwt", false, nil
}, option, []byte("plain"))
require.Error(t, err)
mu.Lock()
defer mu.Unlock()
// One assignment (no reassignment), and only the inner bounded same-FID
// attempts reached the server (the outer loop did not re-enter).
require.Equal(t, 1, assigns)
require.Equal(t, 3, calls)
}
// Generic401 still uses the old RetryForever behavior. Only the explicitly
// marked expiry path gains the separate finite recovery budget.
func TestUploadExpiryPreservesRetryForeverGenericDenial(t *testing.T) {
var mu sync.Mutex
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
n := calls
mu.Unlock()
if n <= 3 {
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
return
}
_, _ = io.WriteString(w, `{"size":5}`)
}))
defer srv.Close()
u := NewUploaderWithHttpClient(srv.Client())
assigns := 0
_, result, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) {
assigns++
return "1,0000000100001234", strings.TrimPrefix(srv.URL, "http://"), "disposable", false, nil
}, &UploadOption{RetryForever: true}, []byte("plain"))
require.NoError(t, err)
require.Equal(t, uint32(5), result.Size)
require.Equal(t, 2, assigns)
mu.Lock()
actual := calls
mu.Unlock()
require.Equal(t, 4, actual)
}
func TestUploadExpiryCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var mu sync.Mutex
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
mu.Unlock()
cancel()
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
}))
defer srv.Close()
client := srv.Client()
client.Timeout = 2 * time.Second
_, err := NewUploaderWithHttpClient(client).UploadData(ctx, []byte("plain"), &UploadOption{UploadUrl: srv.URL})
require.ErrorIs(t, err, context.Canceled)
mu.Lock()
actual := calls
mu.Unlock()
require.Equal(t, 1, actual)
}
func TestUploadExpiryTruncatedResponsePreservesMarker(t *testing.T) {
for _, marked := range []bool{false, true} {
name := "unmarked"
if marked {
name = "marked"
}
t.Run(name, func(t *testing.T) {
var mu sync.Mutex
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
mu.Lock()
calls++
mu.Unlock()
if marked {
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
}
w.Header().Set("Content-Length", "100")
w.WriteHeader(401)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
}))
defer srv.Close()
_, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: srv.URL})
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
var statusErr *uploadStatusError
require.ErrorAs(t, err, &statusErr)
require.Equal(t, 401, statusErr.StatusCode)
require.Equal(t, marked, statusErr.expiredBeforeAppend)
require.Equal(t, marked, ShouldReassignUpload(err))
want := 3
if marked {
want = 1
}
mu.Lock()
actual := calls
mu.Unlock()
require.Equal(t, want, actual)
})
}
}
+5 -1
View File
@@ -148,13 +148,17 @@ func GetJwt(r *http.Request) EncodedJwt {
}
func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims) (token *jwt.Token, err error) {
return decodeJwt(signingKey, tokenString, claims)
}
func decodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims, options ...jwt.ParserOption) (token *jwt.Token, err error) {
// check exp, nbf
return jwt.ParseWithClaims(string(tokenString), claims, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("unknown token method")
}
return []byte(signingKey), nil
})
}, options...)
}
// FilerJwtProvider signs the credential a filer's HTTP API expects. A nil
@@ -0,0 +1,32 @@
package security
import (
"errors"
jwt "github.com/golang-jwt/jwt/v5"
"time"
)
// IsExpiredVolumeJwt reports a signed, FID-bound credential whose only failure
// is expiration. It never authorizes a request; it classifies an existing denial
// so an assignment-owning caller can obtain a fresh credential with a new FID.
func IsExpiredVolumeJwt(key SigningKey, encoded EncodedJwt, expectedFid string) bool {
return isExpiredVolumeJwtAt(key, encoded, expectedFid, time.Now())
}
func isExpiredVolumeJwtAt(key SigningKey, encoded EncodedJwt, expectedFid string, now time.Time) bool {
if len(key) == 0 || encoded == "" {
return false
}
claims := &SeaweedFileIdClaims{}
parsed, err := decodeJwt(key, encoded, claims, jwt.WithTimeFunc(func() time.Time { return now }))
// ParseWithClaims verifies the HMAC signature before validating any claims.
if parsed == nil || !errors.Is(err, jwt.ErrTokenExpired) || claims.Fid != expectedFid {
return false
}
otherClaims := *claims
otherClaims.ExpiresAt = nil
// Do not mistake expired+not-before/issued-at errors for expiry alone. This
// extra issued-at restriction applies only to classification, never normal
// authorization. Keep future volume claim policies aligned here.
return jwt.NewValidator(jwt.WithTimeFunc(func() time.Time { return now }), jwt.WithIssuedAt()).Validate(&otherClaims) == nil
}
@@ -0,0 +1,72 @@
package security
import (
jwt "github.com/golang-jwt/jwt/v5"
"github.com/stretchr/testify/require"
"testing"
"time"
)
func TestVolumeJwtExpiryOnly(t *testing.T) {
now := time.Unix(1900000000, 0)
key := SigningKey("disposable-volume-classifier")
for _, tc := range []struct {
name string
expired, nbf, iat, wrongKey, wrongFid, noExpiry bool
want bool
}{
{name: "expired_only", expired: true, want: true},
{name: "expired_future_nbf", expired: true, nbf: true},
{name: "expired_future_iat", expired: true, iat: true},
{name: "expired_bad_signature", expired: true, wrongKey: true},
{name: "expired_wrong_fid", expired: true, wrongFid: true},
{name: "valid"}, {name: "future_nbf", nbf: true}, {name: "no_expiry", noExpiry: true},
} {
t.Run(tc.name, func(t *testing.T) {
fid := "1,0000000100001234"
if tc.wrongFid {
fid = "1,0000000200001234"
}
exp := now.Add(time.Minute)
if tc.expired {
exp = now.Add(-time.Second)
}
claims := SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}}
if tc.noExpiry {
claims.ExpiresAt = nil
}
if tc.nbf {
claims.NotBefore = jwt.NewNumericDate(now.Add(time.Minute))
}
if tc.iat {
claims.IssuedAt = jwt.NewNumericDate(now.Add(time.Minute))
}
signingKey := key
if tc.wrongKey {
signingKey = SigningKey("different-disposable-key")
}
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey))
require.NoError(t, err)
require.Equal(t, tc.want, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now))
})
}
require.False(t, isExpiredVolumeJwtAt(key, "malformed", "1,0000000100001234", now))
require.False(t, isExpiredVolumeJwtAt(nil, "malformed", "1,0000000100001234", now))
// Bad temporal field types must fail parsing instead of producing a marker.
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"fid": "1,0000000100001234", "exp": now.Add(-time.Minute).Unix(), "nbf": "not-a-date"}).SignedString([]byte(key))
require.NoError(t, err)
require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now))
}
func TestVolumeJwtExpiryAlgorithmBoundary(t *testing.T) {
now := time.Unix(1900000000, 0)
key := SigningKey("disposable-algorithm-test")
claims := SeaweedFileIdClaims{Fid: "1,0000000100001234", RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(now.Add(-time.Second))}}
// Existing DecodeJwt permits the HMAC family, not just HS256.
token, err := jwt.NewWithClaims(jwt.SigningMethodHS512, claims).SignedString([]byte(key))
require.NoError(t, err)
require.True(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), claims.Fid, now))
unsigned, err := jwt.NewWithClaims(jwt.SigningMethodNone, claims).SignedString(jwt.UnsafeAllowNoneSignatureType)
require.NoError(t, err)
require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(unsigned), claims.Fid, now))
}
@@ -0,0 +1,468 @@
package weed_server
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
jwt "github.com/golang-jwt/jwt/v5"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
type expiryAssignRPC struct {
filer_pb.SeaweedFilerClient
assign func() (*filer_pb.AssignVolumeResponse, error)
}
func (c expiryAssignRPC) AssignVolume(_ context.Context, _ *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) {
return c.assign()
}
type expiryFilerClient struct{ rpc expiryAssignRPC }
func (c expiryFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(c.rpc)
}
func (c expiryFilerClient) AdjustedUrl(l *filer_pb.Location) string { return l.Url }
func (c expiryFilerClient) GetDataCenter() string { return "" }
type expiryCountingBody struct {
io.ReadCloser
bytesRead int
}
func (b *expiryCountingBody) Read(p []byte) (int, error) {
n, e := b.ReadCloser.Read(p)
b.bytesRead += n
return n, e
}
func expiryVolume(t *testing.T, key security.SigningKey) *VolumeServer {
t.Helper()
s := newTierTestStore(t, t.TempDir())
require.NoError(t, s.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0))
return &VolumeServer{store: s, guard: security.NewGuard(nil, string(key), 10, "", 0), fileSizeLimitBytes: 4 << 20}
}
func expirySignedToken(t *testing.T, key security.SigningKey, fid string, expires time.Time) string {
t.Helper()
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(expires)}}).SignedString([]byte(key))
require.NoError(t, err)
return token
}
// Assignment returns a genuinely valid ten-second JWT. The HTTP server delays
// only that assignment until its exact expiry boundary before running the real
// PostHandler. This bounded wait reproduces transit/preauthorization delay; it
// makes no CPU-throughput or shared-clock-injection assumption.
func TestExpiredVolumeJwtReassignmentHTTP(t *testing.T) {
for _, tc := range []struct {
name string
forever, cipher bool
}{{"bounded", false, false}, {"bounded_cipher", false, true}, {"retry_forever", true, false}, {"retry_forever_cipher", true, true}} {
name, forever := tc.name, tc.forever
t.Run(name, func(t *testing.T) {
key := security.SigningKey("disposable-volume-expiry-test")
vs := expiryVolume(t, key)
plain := bytes.Repeat([]byte("compressible lifecycle payload"), 4096)
sum := md5.Sum(plain)
var mu sync.Mutex
var ids []string
var statuses []int
var outcomes []string
var bodyReads []int
var fsyncs []string
var assigned int
var firstExpires time.Time
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
expires := firstExpires
mu.Unlock()
if r.URL.Path == "/1,0000000100001234" {
if delay := time.Until(expires.Add(100 * time.Millisecond)); delay > 0 {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-timer.C:
case <-r.Context().Done():
return
}
}
}
counted := &expiryCountingBody{ReadCloser: r.Body}
r.Body = counted
rec := httptest.NewRecorder()
vs.PostHandler(rec, r)
mu.Lock()
ids = append(ids, r.URL.Path)
statuses = append(statuses, rec.Code)
outcomes = append(outcomes, rec.Header().Get("X-Seaweed-Write-Outcome"))
bodyReads = append(bodyReads, counted.bytesRead)
fsyncs = append(fsyncs, r.URL.Query().Get("fsync"))
mu.Unlock()
for k, v := range rec.Header() {
w.Header()[k] = v
}
w.WriteHeader(rec.Code)
_, _ = w.Write(rec.Body.Bytes())
}))
defer server.Close()
c := expiryFilerClient{rpc: expiryAssignRPC{assign: func() (*filer_pb.AssignVolumeResponse, error) {
assigned++
fid := "1,0000000100001234"
expiry := time.Now().Add(10 * time.Second)
if assigned == 1 {
mu.Lock()
firstExpires = expiry
mu.Unlock()
}
if assigned > 1 {
fid = "1,0000000200001234"
expiry = time.Now().Add(time.Hour)
}
token := expirySignedToken(t, key, fid, expiry)
parsed, decodeErr := security.DecodeJwt(key, security.EncodedJwt(token), &security.SeaweedFileIdClaims{})
require.NoError(t, decodeErr)
require.True(t, parsed.Valid)
return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: token, Fsync: true, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}}, nil
}}}
httpClient := server.Client()
httpClient.Timeout = 25 * time.Second
uploader := operation.NewUploaderWithHttpClient(httpClient)
fid, result, err, returned := uploader.UploadWithRetry(c, &filer_pb.AssignVolumeRequest{Path: "/own/long.bin"}, &operation.UploadOption{Cipher: tc.cipher, Filename: "long.bin", MimeType: "application/octet-stream", WantMd5: true, RetryForever: forever}, bytes.NewReader(plain))
require.NoError(t, err)
require.Equal(t, "1,0000000200001234", fid)
require.Equal(t, 2, assigned)
require.Equal(t, plain, returned)
require.Equal(t, uint32(len(plain)), result.Size)
require.Equal(t, base64.StdEncoding.EncodeToString(sum[:]), result.ContentMd5)
mu.Lock()
gotIds := append([]string(nil), ids...)
gotStatuses := append([]int(nil), statuses...)
gotFsyncs := append([]string(nil), fsyncs...)
gotOutcomes := append([]string(nil), outcomes...)
gotBodyReads := append([]int(nil), bodyReads...)
mu.Unlock()
require.Equal(t, []string{"/1,0000000100001234", "/1,0000000200001234"}, gotIds)
require.Equal(t, []int{http.StatusUnauthorized, http.StatusCreated}, gotStatuses)
require.Equal(t, []string{"true", "true"}, gotFsyncs)
require.Equal(t, []string{"expired-jwt-before-append", ""}, gotOutcomes)
require.Equal(t, 0, gotBodyReads[0])
require.Positive(t, gotBodyReads[1])
require.Equal(t, uint32(1), result.Gzip)
if tc.cipher {
require.NotEmpty(t, result.CipherKey)
} else {
require.Empty(t, result.CipherKey)
}
_, err = vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil)
require.Error(t, err)
stored := &needle.Needle{Id: 2, Cookie: 0x1234}
_, err = vs.store.ReadVolumeNeedle(1, stored, nil, nil)
require.NoError(t, err)
require.NotEmpty(t, stored.Data)
recovered := stored.Data
if tc.cipher {
recovered, err = util.Decrypt(recovered, util.CipherKey(result.CipherKey))
require.NoError(t, err)
}
if result.Gzip > 0 {
recovered, err = util.DecompressData(recovered)
require.NoError(t, err)
}
require.Equal(t, plain, recovered)
})
}
}
func TestVolumeJwtExpiryRefusalBoundary(t *testing.T) {
key := security.SigningKey("disposable-volume-boundary")
for _, tc := range []struct {
name string
expired, nbf, iat, badKey, badFid, missing, urlencoded, suffix, noKey bool
want int
marker bool
}{
{name: "expired", expired: true, want: 401, marker: true},
{name: "expired_suffix", expired: true, suffix: true, want: 401, marker: true},
{name: "expired_future_nbf", expired: true, nbf: true, want: 401},
{name: "expired_future_iat", expired: true, iat: true, want: 401},
{name: "expired_wrong_signature", expired: true, badKey: true, want: 401},
{name: "expired_wrong_fid", expired: true, badFid: true, want: 401},
{name: "missing", missing: true, want: 401},
{name: "nonmultipart", expired: true, urlencoded: true, want: 401},
{name: "valid", want: 201},
{name: "no_key", missing: true, noKey: true, want: 201},
} {
t.Run(tc.name, func(t *testing.T) {
vs := expiryVolume(t, key)
if tc.noKey {
vs.guard = security.NewGuard(nil, "", 0, "", 0)
}
plain := []byte("boundary content")
var body bytes.Buffer
mw := multipart.NewWriter(&body)
part, e := mw.CreateFormFile("file", "sample.bin")
require.NoError(t, e)
_, e = part.Write(plain)
require.NoError(t, e)
require.NoError(t, mw.Close())
contentType := mw.FormDataContentType()
if tc.urlencoded {
body.Reset()
body.WriteString("sample=value")
contentType = "application/x-www-form-urlencoded"
}
counted := &expiryCountingBody{ReadCloser: io.NopCloser(bytes.NewReader(body.Bytes()))}
path := "/1,0000000100001234"
if tc.suffix {
path += "_2"
}
req := httptest.NewRequest(http.MethodPost, path, counted)
req.Header.Set("Content-Type", contentType)
// A request-supplied marker must never be echoed as response proof.
req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append")
if !tc.missing {
fid := "1,0000000100001234"
if tc.badFid {
fid = "1,0000000200001234"
}
exp := time.Now().Add(time.Hour)
if tc.expired {
exp = time.Now().Add(-time.Hour)
}
claims := security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}}
if tc.nbf {
claims.NotBefore = jwt.NewNumericDate(time.Now().Add(time.Hour))
}
if tc.iat {
claims.IssuedAt = jwt.NewNumericDate(time.Now().Add(time.Hour))
}
signingKey := key
if tc.badKey {
signingKey = security.SigningKey("different-disposable-key")
}
token, e := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey))
require.NoError(t, e)
req.Header.Set("Authorization", "Bearer "+token)
}
rec := httptest.NewRecorder()
vs.PostHandler(rec, req)
require.Equal(t, tc.want, rec.Code)
want := ""
if tc.marker {
want = "expired-jwt-before-append"
}
require.Equal(t, want, rec.Header().Get("X-Seaweed-Write-Outcome"))
n := &needle.Needle{Id: 1, Cookie: 0x1234}
_, readErr := vs.store.ReadVolumeNeedle(1, n, nil, nil)
if tc.want == 401 {
require.Error(t, readErr)
if !tc.urlencoded {
require.Zero(t, counted.bytesRead)
}
} else {
require.NoError(t, readErr)
require.Equal(t, plain, n.Data)
}
})
}
}
func TestVolumeJwtExpiryProxyForwarding(t *testing.T) {
volumeKey := security.SigningKey("disposable-proxy-volume-key")
filerKey := security.SigningKey("disposable-proxy-filer-key")
vs := expiryVolume(t, volumeKey)
var hits atomic.Int32
volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) == 1 {
claims := &security.SeaweedFileIdClaims{}
parsed, err := security.DecodeJwt(volumeKey, security.GetJwt(r), claims)
if err != nil || !parsed.Valid || claims.ExpiresAt == nil {
w.WriteHeader(500)
return
}
timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond)))
defer timer.Stop()
select {
case <-timer.C:
case <-r.Context().Done():
return
}
}
vs.PostHandler(w, r)
}))
defer volume.Close()
fs := &FilerServer{filerGuard: security.NewGuard(nil, string(filerKey), 3600, "", 0), volumeGuard: security.NewGuard(nil, string(volumeKey), 10, "", 0)}
fid := "1,0000000100001234"
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Exercise the same real caller verifier as filerHandler, then its actual
// forwarding function with an already-resolved synthetic target.
if _, ok := fs.authenticateFilerJwt(r, true); !ok {
w.WriteHeader(401)
return
}
fs.proxyToVolumeServerURL(w, r, fid, volume.URL+"/"+fid)
}))
defer proxy.Close()
caller := string(security.GenJwtForFilerServer(filerKey, 3600))
for i, want := range []int{401, 401, 201} {
var body bytes.Buffer
mw := multipart.NewWriter(&body)
part, err := mw.CreateFormFile("file", "proxy.bin")
require.NoError(t, err)
_, err = part.Write([]byte("proxy clear data"))
require.NoError(t, err)
require.NoError(t, mw.Close())
req, err := http.NewRequest(http.MethodPost, proxy.URL+"/?proxyChunkId="+fid, bytes.NewReader(body.Bytes()))
require.NoError(t, err)
req.Header.Set("Content-Type", mw.FormDataContentType())
req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append")
token := caller
if i == 1 {
token = "invalid-disposable-caller"
}
req.Header.Set("Authorization", "Bearer "+token)
client := proxy.Client()
client.Timeout = 25 * time.Second
resp, err := client.Do(req)
require.NoError(t, err)
_, err = io.Copy(io.Discard, resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, want, resp.StatusCode)
marker := ""
if i == 0 {
marker = "expired-jwt-before-append"
}
require.Equal(t, marker, resp.Header.Get("X-Seaweed-Write-Outcome"))
if i == 1 {
require.Equal(t, int32(1), hits.Load())
}
}
require.Equal(t, int32(2), hits.Load())
}
func TestVolumeJwtExpiryFanoutRollback(t *testing.T) {
key := security.SigningKey("disposable-fanout-expiry-key")
primary, replica := expiryVolume(t, key), expiryVolume(t, key)
firstCommitted := make(chan struct{})
var committed sync.Once
var mu sync.Mutex
var firstReplicaStatuses []int
var firstReplicaMarkers []string
a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
primary.DeleteHandler(w, r)
return
}
rec := httptest.NewRecorder()
primary.PostHandler(rec, r)
if r.URL.Path == "/1,0000000100001234" && rec.Code == 201 {
committed.Do(func() { close(firstCommitted) })
}
for k, v := range rec.Header() {
w.Header()[k] = v
}
w.WriteHeader(rec.Code)
_, _ = w.Write(rec.Body.Bytes())
}))
defer a.Close()
b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
replica.DeleteHandler(w, r)
return
}
if r.URL.Path == "/1,0000000100001234" {
select {
case <-firstCommitted:
case <-r.Context().Done():
return
}
claims := &security.SeaweedFileIdClaims{}
parsed, err := security.DecodeJwt(key, security.GetJwt(r), claims)
if err != nil || !parsed.Valid || claims.ExpiresAt == nil {
w.WriteHeader(500)
return
}
timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond)))
defer timer.Stop()
select {
case <-timer.C:
case <-r.Context().Done():
return
}
}
rec := httptest.NewRecorder()
replica.PostHandler(rec, r)
if r.URL.Path == "/1,0000000100001234" {
mu.Lock()
firstReplicaStatuses = append(firstReplicaStatuses, rec.Code)
firstReplicaMarkers = append(firstReplicaMarkers, rec.Header().Get("X-Seaweed-Write-Outcome"))
mu.Unlock()
}
for k, v := range rec.Header() {
w.Header()[k] = v
}
w.WriteHeader(rec.Code)
_, _ = w.Write(rec.Body.Bytes())
}))
defer b.Close()
var assignments int
plain := bytes.Repeat([]byte("fanout content"), 4096)
ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
defer cancel()
result, err := operation.UploadReaderInChunks(ctx, bytes.NewReader(plain), &operation.ChunkedUploadOption{
ChunkSize: 1 << 20, MimeType: "application/octet-stream", AssignFunc: func(_ context.Context, _ int, _ uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
assignments++
fid := "1,0000000100001234"
if assignments > 1 {
fid = "1,0000000200001234"
}
return nil, &operation.AssignResult{Fid: fid, Url: strings.TrimPrefix(a.URL, "http://"), Auth: security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(10*time.Second))), Replicas: []operation.Location{{Url: strings.TrimPrefix(b.URL, "http://")}}, Fsync: true}, nil
}, LookupJwtFunc: func(_ context.Context, fid string) security.EncodedJwt {
return security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(time.Hour)))
},
})
require.NoError(t, err)
require.Equal(t, 2, assignments)
mu.Lock()
gotStatuses := append([]int(nil), firstReplicaStatuses...)
gotMarkers := append([]string(nil), firstReplicaMarkers...)
mu.Unlock()
require.Equal(t, []int{401}, gotStatuses)
require.Equal(t, []string{"expired-jwt-before-append"}, gotMarkers)
require.Len(t, result.FileChunks, 1)
require.Equal(t, "1,0000000200001234", result.FileChunks[0].FileId)
for _, vs := range []*VolumeServer{primary, replica} {
_, err := vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil)
require.Error(t, err)
n := &needle.Needle{Id: 2, Cookie: 0x1234}
_, err = vs.store.ReadVolumeNeedle(1, n, nil, nil)
require.NoError(t, err)
data := n.Data
if result.FileChunks[0].IsCompressed {
data, err = util.DecompressData(data)
require.NoError(t, err)
}
require.Equal(t, plain, data)
}
}
+12 -8
View File
@@ -330,19 +330,23 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req
}
func (vs *VolumeServer) maybeCheckJwtAuthorization(r *http.Request, vid, fid string, isWrite bool) bool {
var signingKey security.SigningKey
if isWrite {
signingKey = vs.guard.SigningKey()
if len(signingKey) == 0 {
return true
}
} else {
signingKey = vs.guard.ReadSigningKey()
if len(signingKey) == 0 {
return true
}
}
return vs.checkJwtAuthorizationWithKey(r, vid, fid, signingKey)
}
// checkJwtAuthorizationWithKey authorizes against a caller-snapshotted signing
// key, so a write handler can classify an expired-JWT refusal against the exact
// key it authorized under rather than re-reading one that may have rotated
// between the two reads.
func (vs *VolumeServer) checkJwtAuthorizationWithKey(r *http.Request, vid, fid string, signingKey security.SigningKey) bool {
if len(signingKey) == 0 {
return true
}
tokenStr := security.GetJwt(r)
+20 -2
View File
@@ -7,6 +7,7 @@ import (
"encoding/base64"
"errors"
"fmt"
"mime"
"net/http"
"strconv"
"strings"
@@ -14,11 +15,13 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/topology"
"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
var jpegStart = []byte{0xFF, 0xD8, 0xFF}
@@ -52,7 +55,22 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
return
}
if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
// Snapshot the write signing key once, so the authorization check and the
// expired-JWT classification below read the same key even if it rotates.
writeSigningKey := vs.guard.SigningKey()
if !vs.checkJwtAuthorizationWithKey(r, vid, fid, writeSigningKey) {
// ParseForm does not read multipart bodies. Preserve its existing placement
// and status precedence for other content types, which get no retry marker.
contentType, _, typeErr := mime.ParseMediaType(r.Header.Get("Content-Type"))
if typeErr == nil && contentType == "multipart/form-data" {
baseFid := fid
if sep := strings.LastIndex(baseFid, "_"); sep > 0 {
baseFid = baseFid[:sep]
}
if security.IsExpiredVolumeJwt(writeSigningKey, security.GetJwt(r), vid+","+baseFid) {
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend)
}
}
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
return
}
@@ -77,7 +95,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
status = http.StatusConflict
}
if errors.Is(writeError, topology.ErrorWriteRefused) {
w.Header().Set("X-Seaweed-Write-Outcome", "refused-before-append")
w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeRefusedBeforeAppend)
}
writeJsonError(w, r, status, writeError)
return
+9
View File
@@ -0,0 +1,9 @@
package http
// Write outcomes describe a refusal of this HTTP attempt before append, not the
// absence of a stored needle from an earlier attempt with a lost response.
const (
WriteOutcomeHeader = "X-Seaweed-Write-Outcome"
WriteOutcomeRefusedBeforeAppend = "refused-before-append"
WriteOutcomeExpiredJwtBeforeAppend = "expired-jwt-before-append"
)