diff --git a/seaweed-volume/src/security.rs b/seaweed-volume/src/security.rs index de4168a6fa..162c73a961 100644 --- a/seaweed-volume/src/security.rs +++ b/seaweed-volume/src/security.rs @@ -106,6 +106,39 @@ pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result bool { + match decode_jwt(signing_key, token) { + Err(JwtError::Jwt(e)) + if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {} + _ => return false, + } + + #[derive(Deserialize)] + struct ExpiredClaims { + fid: Option, + iat: Option, + } + let mut validation = Validation::new(Algorithm::HS256); + validation.required_spec_claims.clear(); + validation.validate_exp = false; + validation.validate_nbf = true; + validation.leeway = 0; + let Ok(data) = decode::( + token, + &DecodingKey::from_secret(&signing_key.0), + &validation, + ) else { + return false; + }; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now) +} + // ============================================================================ // Guard // ============================================================================ @@ -410,6 +443,53 @@ mod tests { assert!(decode_jwt(&key2, &token).is_err()); } + #[test] + fn test_is_expired_volume_jwt_classifies_expiry_only() { + let key = SigningKey::from_string("write-key"); + let fid = "3,01637037d6"; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let sign = |claims: serde_json::Value, key: &SigningKey| { + encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(&key.0), + ) + .unwrap() + }; + let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key); + assert!(is_expired_volume_jwt(&key, &expired, fid)); + assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7")); + assert!(!is_expired_volume_jwt( + &SigningKey::from_string("other-key"), + &expired, + fid + )); + assert!(!is_expired_volume_jwt(&SigningKey(vec![]), &expired, fid)); + let immature = sign( + serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}), + &key, + ); + assert!(!is_expired_volume_jwt(&key, &immature, fid)); + let future_issued = sign( + serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}), + &key, + ); + assert!(!is_expired_volume_jwt(&key, &future_issued, fid)); + let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key); + assert!(!is_expired_volume_jwt(&key, &live, fid)); + let unsigned = format!( + "eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.", + base64::Engine::encode( + &base64::engine::general_purpose::URL_SAFE_NO_PAD, + serde_json::json!({"fid": fid, "exp": now - 10}).to_string() + ) + ); + assert!(!is_expired_volume_jwt(&key, &unsigned, fid)); + } + #[test] fn test_guard_empty_whitelist() { let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0); diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index d63cce3ea7..fad351c545 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -483,6 +483,18 @@ fn extract_file_id(path: &str) -> String { format!("{},{}", parsed.vid, fid) } +fn is_multipart_form_data(headers: &HeaderMap) -> bool { + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .is_some_and(|media_type| { + media_type + .trim() + .eq_ignore_ascii_case("multipart/form-data") + }) +} + fn streaming_chunk_size(read_buffer_size_bytes: usize, data_size: usize) -> usize { std::cmp::min( read_buffer_size_bytes.max(DEFAULT_STREAMING_CHUNK_SIZE), @@ -2777,14 +2789,28 @@ pub async fn post_handler( // JWT check for writes let file_id = extract_file_id(&path); let token = extract_jwt(&headers, request.uri()); - if state - .guard - .read() - .unwrap() - .check_jwt_for_file(token.as_deref(), &file_id, true) - .is_err() - { - return json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query)); + let expired_jwt = { + let guard = state.guard.read().unwrap(); + match guard.check_jwt_for_file(token.as_deref(), &file_id, true) { + Ok(()) => None, + Err(_) => Some( + is_multipart_form_data(&headers) + && token.as_deref().is_some_and(|token| { + crate::security::is_expired_volume_jwt(&guard.signing_key, token, &file_id) + }), + ), + } + }; + if let Some(expired_jwt) = expired_jwt { + let mut response = + json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query)); + if expired_jwt { + response.headers_mut().insert( + "x-seaweed-write-outcome", + header::HeaderValue::from_static("expired-jwt-before-append"), + ); + } + return response; } // Upload throttling: check inflight bytes against limit @@ -5465,6 +5491,83 @@ mod tests { assert_eq!(extract_jwt(&headers, &uri), None); } + #[tokio::test] + async fn http_expired_write_jwt_is_marked_before_append() { + use crate::security::SigningKey; + use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let state = volume_test_state(&tmp); + let key = SigningKey(b"write-secret".to_vec()); + state.guard.write().unwrap().signing_key = key.clone(); + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let before = volume.dat_file_size().unwrap(); + let fid = format!("1,{:x}{TEST_COOKIE:08x}", 0x6e7a_0c09u64); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let sign = |key: &[u8]| { + encode( + &Header::new(Algorithm::HS256), + &serde_json::json!({"fid": fid, "exp": now - 10}), + &EncodingKey::from_secret(key), + ) + .unwrap() + }; + let boundary = "expiredjwtboundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a\"\r\n\r\nexpired-bytes\r\n--{boundary}--\r\n" + ); + for (token, content_type, marked) in [ + ( + sign(&key.0), + format!("multipart/form-data; boundary={boundary}"), + true, + ), + ( + sign(b"other-secret"), + format!("multipart/form-data; boundary={boundary}"), + false, + ), + (sign(&key.0), "application/octet-stream".to_string(), false), + ] { + let response = super::super::volume_server::build_admin_router(state.clone()) + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/{fid}")) + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(header::CONTENT_TYPE, &content_type) + .body(Body::from(body.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{content_type}" + ); + assert_eq!( + response + .headers() + .get("x-seaweed-write-outcome") + .map(|value| value.to_str().unwrap()), + marked.then_some("expired-jwt-before-append"), + "{content_type}" + ); + } + assert_eq!(volume.dat_file_size().unwrap(), before); + } + #[test] fn test_handle_range_single() { let data = b"hello world"; diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 0b26dc192e..e69f6b283c 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -131,6 +131,7 @@ var ( type uploadStatusError struct { StatusCode int refusedBeforeAppend bool + expiredBeforeAppend bool stalled bool // the stall watchdog cancelled this attempt's request err error } @@ -149,7 +150,7 @@ func ShouldReassignUpload(err error) bool { } var se *uploadStatusError if errors.As(err, &se) { - return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend) + return se.StatusCode == 0 || se.StatusCode >= 500 || (se.StatusCode == http.StatusConflict && se.refusedBeforeAppend) || (se.StatusCode == http.StatusUnauthorized && se.expiredBeforeAppend) } return strings.Contains(err.Error(), "transport") } @@ -243,17 +244,37 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho return err } + // A credential can expire while the first HTTP request is queued. Bound + // expiry-specific reassignment even when unrelated transient retries are + // intentionally unbounded. The third marked refusal is final. + expiryRefusals := 0 + shouldRetry := func(err error) bool { + var statusErr *uploadStatusError + if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend { + // A proxy upload already exhausted its bounded same-FID retries in + // retriedUploadData (with a fresh filer credential each time); the + // filer owns the FID, so a reassignment here is wrong, and counting + // exhausted groups would also inflate the refusal budget. Make it + // terminal; only direct uploads reassign. + if util_http.IsProxyChunkUrl(uploadOption.UploadUrl) { + return false + } + expiryRefusals++ + return expiryRefusals < 3 + } + return ShouldReassignUpload(err) + } if uploadOption.RetryForever { util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) { glog.V(0).Infof("upload content: %v", err) var statusErr *uploadStatusError - if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusConflict { - return ShouldReassignUpload(err) + if errors.As(err, &statusErr) && (statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend)) { + return shouldRetry(err) } return true }) } else { - err = util.RetryOnError("uploadWithRetry", ShouldReassignUpload, doUploadFunc) + err = util.RetryOnError("uploadWithRetry", shouldRetry, doUploadFunc) } return @@ -380,9 +401,26 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *prepa } var statusErr *uploadStatusError if errors.As(err, &statusErr) { - if statusErr.StatusCode == http.StatusConflict { + // A marked-expiry 401 is terminal here only for a direct volume + // upload: its assignment JWT is spent, so recovery needs a fresh + // assignment (uploadWithRetryData). A filer-proxy upload instead + // mints a fresh volume JWT on each forwarded request, so a bounded + // same-FID retry is enough and reassignment is both unnecessary and + // wrong (the proxy owns the FID). Keep that same-FID retry for it. + if statusErr.StatusCode == http.StatusConflict || (statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && !util_http.IsProxyChunkUrl(option.UploadUrl)) { return nil, err } + if statusErr.StatusCode == http.StatusUnauthorized && statusErr.expiredBeforeAppend && util_http.IsProxyChunkUrl(option.UploadUrl) { + // The caller's filer credential was minted once and may have + // expired alongside the volume JWT (both default to 10s), so + // re-mint it before replaying the same FID; otherwise the filer + // rejects the retry with an unmarked 401 and recovery stops. + if option.FilerJwt != nil { + option.Jwt = option.FilerJwt(true) + } else { + option.Jwt = security.EncodedJwt(util_http.JwtForFilerServer(true)) + } + } if statusErr.stalled { option.stallWatchdogDisarmed = true } @@ -720,26 +758,27 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction respStallTimer := time.AfterFunc(uploadStallMinWait, stallFired) defer respStallTimer.Stop() } - refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get("X-Seaweed-Write-Outcome") == "refused-before-append" + refusedBeforeAppend := resp.StatusCode == http.StatusConflict && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeRefusedBeforeAppend + expiredBeforeAppend := resp.StatusCode == http.StatusUnauthorized && resp.Header.Get(util_http.WriteOutcomeHeader) == util_http.WriteOutcomeExpiredJwtBeforeAppend resp_body, ra_err := io.ReadAll(resp.Body) uploadDone.Store(true) if ra_err != nil { stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() - return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} + return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} } unmarshal_err := json.Unmarshal(resp_body, &ret) if unmarshal_err != nil { stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body)) - return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)} + return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)} } - if resp.StatusCode == http.StatusConflict && ret.Error == "" { + if (resp.StatusCode == http.StatusConflict || expiredBeforeAppend) && ret.Error == "" { ret.Error = http.StatusText(resp.StatusCode) } if ret.Error != "" { stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() - return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)} + return nil, &uploadStatusError{StatusCode: resp.StatusCode, refusedBeforeAppend: refusedBeforeAppend, expiredBeforeAppend: expiredBeforeAppend, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)} } if bps := uploadObservedBps(bodySize, movedIn); bps > 0 { uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps}) diff --git a/weed/operation/volume_jwt_expiry_recovery_test.go b/weed/operation/volume_jwt_expiry_recovery_test.go new file mode 100644 index 0000000000..dbae41af6c --- /dev/null +++ b/weed/operation/volume_jwt_expiry_recovery_test.go @@ -0,0 +1,357 @@ +package operation + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/security" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" + "github.com/stretchr/testify/require" +) + +func TestUploadExpiryResponseContract(t *testing.T) { + for _, tc := range []struct { + name string + status int + marker, body string + want bool + calls int + }{ + {"marked", 401, "expired-jwt-before-append", `{"error":"wrong jwt"}`, true, 1}, + {"marked_empty_error", 401, "expired-jwt-before-append", `{}`, true, 1}, + {"marked_bad_json", 401, "expired-jwt-before-append", `{`, true, 1}, + {"unmarked", 401, "", `{"error":"wrong jwt"}`, false, 3}, + {"wrong_value", 401, "expired-jwt-before-append-extra", `{"error":"wrong jwt"}`, false, 3}, + {"wrong_status", 403, "expired-jwt-before-append", `{"error":"denied"}`, false, 3}, + {"existing_409", 409, "refused-before-append", `{"error":"full"}`, true, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + mu.Unlock() + w.Header().Set(util_http.WriteOutcomeHeader, tc.marker) + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + })) + defer srv.Close() + u := NewUploaderWithHttpClient(srv.Client()) + _, err := u.UploadData(context.Background(), []byte("direct preassigned data"), &UploadOption{UploadUrl: srv.URL, Jwt: "disposable-preassigned"}) + require.Error(t, err) + require.Equal(t, tc.want, ShouldReassignUpload(err)) + mu.Lock() + actual := calls + mu.Unlock() + require.Equal(t, tc.calls, actual) + }) + } +} + +func TestUploadExpiryBudgetAndAssignmentFailure(t *testing.T) { + for _, tc := range []struct { + name string + forever, assignmentFailure bool + }{{"bounded", false, false}, {"forever", true, false}, {"assignment_failure", false, true}} { + t.Run(tc.name, func(t *testing.T) { + var mu sync.Mutex + calls := 0 + var payloads [][]byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mr, err := r.MultipartReader() + if err != nil { + w.WriteHeader(400) + return + } + part, err := mr.NextPart() + if err != nil { + w.WriteHeader(400) + return + } + data, err := io.ReadAll(part) + if err != nil { + w.WriteHeader(400) + return + } + mu.Lock() + calls++ + n := calls + payloads = append(payloads, data) + mu.Unlock() + // Terminal fallback: if the expiry budget regressed and did not + // stop at 3, end the run with an unmarked 409 so this test fails + // here instead of looping to the package timeout under RetryForever. + if n > 3 { + w.WriteHeader(409) + _, _ = io.WriteString(w, `{"error":"full"}`) + return + } + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + })) + defer srv.Close() + u := NewUploaderWithHttpClient(srv.Client()) + assigns := 0 + sentinel := errors.New("synthetic assignment failure") + _, _, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) { + assigns++ + if tc.assignmentFailure && assigns == 2 { + return "", "", "", false, sentinel + } + return fmt.Sprintf("1,%08x00001234", assigns), strings.TrimPrefix(srv.URL, "http://"), "disposable-per-assignment", false, nil + }, &UploadOption{Cipher: true, MimeType: "application/octet-stream", RetryForever: tc.forever}, bytes.Repeat([]byte("compressible data"), 1024)) + require.Error(t, err) + mu.Lock() + actual := calls + got := append([][]byte(nil), payloads...) + mu.Unlock() + if tc.assignmentFailure { + require.ErrorIs(t, err, sentinel) + require.Equal(t, 2, assigns) + require.Equal(t, 1, actual) + } else { + require.Equal(t, 3, assigns) + require.Equal(t, 3, actual) + // The run must end on the 3rd marked 401 (the budget), not the + // 409 terminal fallback above. + var finalErr *uploadStatusError + require.ErrorAs(t, err, &finalErr) + require.Equal(t, http.StatusUnauthorized, finalErr.StatusCode) + require.True(t, finalErr.expiredBeforeAppend) + } + require.NotEmpty(t, got) + for _, data := range got { + require.Equal(t, got[0], data) + } + }) + } +} + +// A filer-proxy upload must retry a marked-expiry 401 on the SAME FID -- the +// proxy mints a fresh volume JWT per forwarded request, so a same-FID replay +// recovers without a reassignment it does not own. +func TestUploadExpiryProxyRetriesSameFid(t *testing.T) { + const fid = "3,01637037d6" + var mu sync.Mutex + calls := 0 + var seenFids []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + n := calls + seenFids = append(seenFids, r.URL.Query().Get(util_http.ProxyChunkIdParam)) + mu.Unlock() + if n == 1 { + // The proxy-minted volume JWT expired in transit on the first try. + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + return + } + _, _ = io.WriteString(w, `{"size":5}`) + })) + defer srv.Close() + proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid + result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: proxyURL}) + require.NoError(t, err) + require.Equal(t, uint32(5), result.Size) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 2, calls) + // Both forwarded requests used the same FID: a proxy upload never reassigns. + require.Equal(t, []string{fid, fid}, seenFids) +} + +// With filer signing enabled, the caller's filer credential can expire together +// with the volume JWT. The proxy same-FID retry must carry a freshly minted +// filer credential, not replay the stale one (which the filer would reject with +// an unmarked 401 that stops recovery). +func TestUploadExpiryProxyRefreshesFilerJwtOnRetry(t *testing.T) { + const fid = "3,01637037d6" + var mu sync.Mutex + calls := 0 + var auths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + n := calls + auths = append(auths, r.Header.Get("Authorization")) + mu.Unlock() + if n == 1 { + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + return + } + _, _ = io.WriteString(w, `{"size":5}`) + })) + defer srv.Close() + proxyURL := srv.URL + "/?" + util_http.ProxyChunkIdParam + "=" + fid + minted := 0 + option := &UploadOption{UploadUrl: proxyURL, Jwt: "stale-filer-jwt", FilerJwt: func(isWrite bool) security.EncodedJwt { + minted++ + return security.EncodedJwt(fmt.Sprintf("fresh-filer-jwt-%d", minted)) + }} + result, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), option) + require.NoError(t, err) + require.Equal(t, uint32(5), result.Size) + mu.Lock() + defer mu.Unlock() + require.Equal(t, 2, calls) + require.Equal(t, 1, minted) + // The first request carried the stale credential; the retry re-minted one. + require.Equal(t, "Bearer stale-filer-jwt", auths[0]) + require.Equal(t, "Bearer fresh-filer-jwt-1", auths[1]) +} + +// A proxy upload driven through uploadWithRetryData must be terminal after its +// bounded same-FID attempts: the outer loop must not reassign a FID the proxy +// owns, nor count exhausted groups toward the expiry budget. +func TestUploadExpiryProxyDoesNotReassignThroughOuterRetry(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + mu.Unlock() + // The proxy keeps reporting the marked expiry (never recovers). + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + })) + defer srv.Close() + host := strings.TrimPrefix(srv.URL, "http://") + assigns := 0 + option := &UploadOption{ + FilerJwt: func(isWrite bool) security.EncodedJwt { return "fresh-filer-jwt" }, + GenUploadUrl: func(host, fileId string) string { + return "http://" + host + "/?" + util_http.ProxyChunkIdParam + "=" + fileId + }, + } + _, _, err := NewUploaderWithHttpClient(srv.Client()).uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) { + assigns++ + return "3,01637037d6", host, "vol-jwt", false, nil + }, option, []byte("plain")) + require.Error(t, err) + mu.Lock() + defer mu.Unlock() + // One assignment (no reassignment), and only the inner bounded same-FID + // attempts reached the server (the outer loop did not re-enter). + require.Equal(t, 1, assigns) + require.Equal(t, 3, calls) +} + +// Generic401 still uses the old RetryForever behavior. Only the explicitly +// marked expiry path gains the separate finite recovery budget. +func TestUploadExpiryPreservesRetryForeverGenericDenial(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n <= 3 { + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + return + } + _, _ = io.WriteString(w, `{"size":5}`) + })) + defer srv.Close() + u := NewUploaderWithHttpClient(srv.Client()) + assigns := 0 + _, result, err := u.uploadWithRetryData(func() (string, string, security.EncodedJwt, bool, error) { + assigns++ + return "1,0000000100001234", strings.TrimPrefix(srv.URL, "http://"), "disposable", false, nil + }, &UploadOption{RetryForever: true}, []byte("plain")) + require.NoError(t, err) + require.Equal(t, uint32(5), result.Size) + require.Equal(t, 2, assigns) + mu.Lock() + actual := calls + mu.Unlock() + require.Equal(t, 4, actual) +} + +func TestUploadExpiryCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + mu.Unlock() + cancel() + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + })) + defer srv.Close() + client := srv.Client() + client.Timeout = 2 * time.Second + _, err := NewUploaderWithHttpClient(client).UploadData(ctx, []byte("plain"), &UploadOption{UploadUrl: srv.URL}) + require.ErrorIs(t, err, context.Canceled) + mu.Lock() + actual := calls + mu.Unlock() + require.Equal(t, 1, actual) +} + +func TestUploadExpiryTruncatedResponsePreservesMarker(t *testing.T) { + for _, marked := range []bool{false, true} { + name := "unmarked" + if marked { + name = "marked" + } + t.Run(name, func(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + mu.Lock() + calls++ + mu.Unlock() + if marked { + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + } + w.Header().Set("Content-Length", "100") + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + })) + defer srv.Close() + _, err := NewUploaderWithHttpClient(srv.Client()).UploadData(context.Background(), []byte("plain"), &UploadOption{UploadUrl: srv.URL}) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + var statusErr *uploadStatusError + require.ErrorAs(t, err, &statusErr) + require.Equal(t, 401, statusErr.StatusCode) + require.Equal(t, marked, statusErr.expiredBeforeAppend) + require.Equal(t, marked, ShouldReassignUpload(err)) + want := 3 + if marked { + want = 1 + } + mu.Lock() + actual := calls + mu.Unlock() + require.Equal(t, want, actual) + }) + } +} diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 97fb9ac9fc..f2d51066c9 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -148,13 +148,17 @@ func GetJwt(r *http.Request) EncodedJwt { } func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims) (token *jwt.Token, err error) { + return decodeJwt(signingKey, tokenString, claims) +} + +func decodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims, options ...jwt.ParserOption) (token *jwt.Token, err error) { // check exp, nbf return jwt.ParseWithClaims(string(tokenString), claims, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("unknown token method") } return []byte(signingKey), nil - }) + }, options...) } // FilerJwtProvider signs the credential a filer's HTTP API expects. A nil diff --git a/weed/security/volume_jwt_expiry_recovery.go b/weed/security/volume_jwt_expiry_recovery.go new file mode 100644 index 0000000000..76c040b318 --- /dev/null +++ b/weed/security/volume_jwt_expiry_recovery.go @@ -0,0 +1,32 @@ +package security + +import ( + "errors" + jwt "github.com/golang-jwt/jwt/v5" + "time" +) + +// IsExpiredVolumeJwt reports a signed, FID-bound credential whose only failure +// is expiration. It never authorizes a request; it classifies an existing denial +// so an assignment-owning caller can obtain a fresh credential with a new FID. +func IsExpiredVolumeJwt(key SigningKey, encoded EncodedJwt, expectedFid string) bool { + return isExpiredVolumeJwtAt(key, encoded, expectedFid, time.Now()) +} + +func isExpiredVolumeJwtAt(key SigningKey, encoded EncodedJwt, expectedFid string, now time.Time) bool { + if len(key) == 0 || encoded == "" { + return false + } + claims := &SeaweedFileIdClaims{} + parsed, err := decodeJwt(key, encoded, claims, jwt.WithTimeFunc(func() time.Time { return now })) + // ParseWithClaims verifies the HMAC signature before validating any claims. + if parsed == nil || !errors.Is(err, jwt.ErrTokenExpired) || claims.Fid != expectedFid { + return false + } + otherClaims := *claims + otherClaims.ExpiresAt = nil + // Do not mistake expired+not-before/issued-at errors for expiry alone. This + // extra issued-at restriction applies only to classification, never normal + // authorization. Keep future volume claim policies aligned here. + return jwt.NewValidator(jwt.WithTimeFunc(func() time.Time { return now }), jwt.WithIssuedAt()).Validate(&otherClaims) == nil +} diff --git a/weed/security/volume_jwt_expiry_recovery_test.go b/weed/security/volume_jwt_expiry_recovery_test.go new file mode 100644 index 0000000000..2511f2f1a4 --- /dev/null +++ b/weed/security/volume_jwt_expiry_recovery_test.go @@ -0,0 +1,72 @@ +package security + +import ( + jwt "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestVolumeJwtExpiryOnly(t *testing.T) { + now := time.Unix(1900000000, 0) + key := SigningKey("disposable-volume-classifier") + for _, tc := range []struct { + name string + expired, nbf, iat, wrongKey, wrongFid, noExpiry bool + want bool + }{ + {name: "expired_only", expired: true, want: true}, + {name: "expired_future_nbf", expired: true, nbf: true}, + {name: "expired_future_iat", expired: true, iat: true}, + {name: "expired_bad_signature", expired: true, wrongKey: true}, + {name: "expired_wrong_fid", expired: true, wrongFid: true}, + {name: "valid"}, {name: "future_nbf", nbf: true}, {name: "no_expiry", noExpiry: true}, + } { + t.Run(tc.name, func(t *testing.T) { + fid := "1,0000000100001234" + if tc.wrongFid { + fid = "1,0000000200001234" + } + exp := now.Add(time.Minute) + if tc.expired { + exp = now.Add(-time.Second) + } + claims := SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}} + if tc.noExpiry { + claims.ExpiresAt = nil + } + if tc.nbf { + claims.NotBefore = jwt.NewNumericDate(now.Add(time.Minute)) + } + if tc.iat { + claims.IssuedAt = jwt.NewNumericDate(now.Add(time.Minute)) + } + signingKey := key + if tc.wrongKey { + signingKey = SigningKey("different-disposable-key") + } + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey)) + require.NoError(t, err) + require.Equal(t, tc.want, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now)) + }) + } + require.False(t, isExpiredVolumeJwtAt(key, "malformed", "1,0000000100001234", now)) + require.False(t, isExpiredVolumeJwtAt(nil, "malformed", "1,0000000100001234", now)) + // Bad temporal field types must fail parsing instead of producing a marker. + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"fid": "1,0000000100001234", "exp": now.Add(-time.Minute).Unix(), "nbf": "not-a-date"}).SignedString([]byte(key)) + require.NoError(t, err) + require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), "1,0000000100001234", now)) +} + +func TestVolumeJwtExpiryAlgorithmBoundary(t *testing.T) { + now := time.Unix(1900000000, 0) + key := SigningKey("disposable-algorithm-test") + claims := SeaweedFileIdClaims{Fid: "1,0000000100001234", RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(now.Add(-time.Second))}} + // Existing DecodeJwt permits the HMAC family, not just HS256. + token, err := jwt.NewWithClaims(jwt.SigningMethodHS512, claims).SignedString([]byte(key)) + require.NoError(t, err) + require.True(t, isExpiredVolumeJwtAt(key, EncodedJwt(token), claims.Fid, now)) + unsigned, err := jwt.NewWithClaims(jwt.SigningMethodNone, claims).SignedString(jwt.UnsafeAllowNoneSignatureType) + require.NoError(t, err) + require.False(t, isExpiredVolumeJwtAt(key, EncodedJwt(unsigned), claims.Fid, now)) +} diff --git a/weed/server/volume_jwt_expiry_recovery_test.go b/weed/server/volume_jwt_expiry_recovery_test.go new file mode 100644 index 0000000000..800d32da5c --- /dev/null +++ b/weed/server/volume_jwt_expiry_recovery_test.go @@ -0,0 +1,468 @@ +package weed_server + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type expiryAssignRPC struct { + filer_pb.SeaweedFilerClient + assign func() (*filer_pb.AssignVolumeResponse, error) +} + +func (c expiryAssignRPC) AssignVolume(_ context.Context, _ *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) { + return c.assign() +} + +type expiryFilerClient struct{ rpc expiryAssignRPC } + +func (c expiryFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error { + return fn(c.rpc) +} +func (c expiryFilerClient) AdjustedUrl(l *filer_pb.Location) string { return l.Url } +func (c expiryFilerClient) GetDataCenter() string { return "" } + +type expiryCountingBody struct { + io.ReadCloser + bytesRead int +} + +func (b *expiryCountingBody) Read(p []byte) (int, error) { + n, e := b.ReadCloser.Read(p) + b.bytesRead += n + return n, e +} + +func expiryVolume(t *testing.T, key security.SigningKey) *VolumeServer { + t.Helper() + s := newTierTestStore(t, t.TempDir()) + require.NoError(t, s.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0)) + return &VolumeServer{store: s, guard: security.NewGuard(nil, string(key), 10, "", 0), fileSizeLimitBytes: 4 << 20} +} + +func expirySignedToken(t *testing.T, key security.SigningKey, fid string, expires time.Time) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(expires)}}).SignedString([]byte(key)) + require.NoError(t, err) + return token +} + +// Assignment returns a genuinely valid ten-second JWT. The HTTP server delays +// only that assignment until its exact expiry boundary before running the real +// PostHandler. This bounded wait reproduces transit/preauthorization delay; it +// makes no CPU-throughput or shared-clock-injection assumption. +func TestExpiredVolumeJwtReassignmentHTTP(t *testing.T) { + for _, tc := range []struct { + name string + forever, cipher bool + }{{"bounded", false, false}, {"bounded_cipher", false, true}, {"retry_forever", true, false}, {"retry_forever_cipher", true, true}} { + name, forever := tc.name, tc.forever + t.Run(name, func(t *testing.T) { + key := security.SigningKey("disposable-volume-expiry-test") + vs := expiryVolume(t, key) + plain := bytes.Repeat([]byte("compressible lifecycle payload"), 4096) + sum := md5.Sum(plain) + var mu sync.Mutex + var ids []string + var statuses []int + var outcomes []string + var bodyReads []int + var fsyncs []string + var assigned int + var firstExpires time.Time + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + expires := firstExpires + mu.Unlock() + if r.URL.Path == "/1,0000000100001234" { + if delay := time.Until(expires.Add(100 * time.Millisecond)); delay > 0 { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-r.Context().Done(): + return + } + } + } + counted := &expiryCountingBody{ReadCloser: r.Body} + r.Body = counted + rec := httptest.NewRecorder() + vs.PostHandler(rec, r) + mu.Lock() + ids = append(ids, r.URL.Path) + statuses = append(statuses, rec.Code) + outcomes = append(outcomes, rec.Header().Get("X-Seaweed-Write-Outcome")) + bodyReads = append(bodyReads, counted.bytesRead) + fsyncs = append(fsyncs, r.URL.Query().Get("fsync")) + mu.Unlock() + for k, v := range rec.Header() { + w.Header()[k] = v + } + w.WriteHeader(rec.Code) + _, _ = w.Write(rec.Body.Bytes()) + })) + defer server.Close() + c := expiryFilerClient{rpc: expiryAssignRPC{assign: func() (*filer_pb.AssignVolumeResponse, error) { + assigned++ + fid := "1,0000000100001234" + expiry := time.Now().Add(10 * time.Second) + if assigned == 1 { + mu.Lock() + firstExpires = expiry + mu.Unlock() + } + if assigned > 1 { + fid = "1,0000000200001234" + expiry = time.Now().Add(time.Hour) + } + token := expirySignedToken(t, key, fid, expiry) + parsed, decodeErr := security.DecodeJwt(key, security.EncodedJwt(token), &security.SeaweedFileIdClaims{}) + require.NoError(t, decodeErr) + require.True(t, parsed.Valid) + return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: token, Fsync: true, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}}, nil + }}} + httpClient := server.Client() + httpClient.Timeout = 25 * time.Second + uploader := operation.NewUploaderWithHttpClient(httpClient) + fid, result, err, returned := uploader.UploadWithRetry(c, &filer_pb.AssignVolumeRequest{Path: "/own/long.bin"}, &operation.UploadOption{Cipher: tc.cipher, Filename: "long.bin", MimeType: "application/octet-stream", WantMd5: true, RetryForever: forever}, bytes.NewReader(plain)) + require.NoError(t, err) + require.Equal(t, "1,0000000200001234", fid) + require.Equal(t, 2, assigned) + require.Equal(t, plain, returned) + require.Equal(t, uint32(len(plain)), result.Size) + require.Equal(t, base64.StdEncoding.EncodeToString(sum[:]), result.ContentMd5) + mu.Lock() + gotIds := append([]string(nil), ids...) + gotStatuses := append([]int(nil), statuses...) + gotFsyncs := append([]string(nil), fsyncs...) + gotOutcomes := append([]string(nil), outcomes...) + gotBodyReads := append([]int(nil), bodyReads...) + mu.Unlock() + require.Equal(t, []string{"/1,0000000100001234", "/1,0000000200001234"}, gotIds) + require.Equal(t, []int{http.StatusUnauthorized, http.StatusCreated}, gotStatuses) + require.Equal(t, []string{"true", "true"}, gotFsyncs) + require.Equal(t, []string{"expired-jwt-before-append", ""}, gotOutcomes) + require.Equal(t, 0, gotBodyReads[0]) + require.Positive(t, gotBodyReads[1]) + require.Equal(t, uint32(1), result.Gzip) + if tc.cipher { + require.NotEmpty(t, result.CipherKey) + } else { + require.Empty(t, result.CipherKey) + } + _, err = vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil) + require.Error(t, err) + stored := &needle.Needle{Id: 2, Cookie: 0x1234} + _, err = vs.store.ReadVolumeNeedle(1, stored, nil, nil) + require.NoError(t, err) + require.NotEmpty(t, stored.Data) + recovered := stored.Data + if tc.cipher { + recovered, err = util.Decrypt(recovered, util.CipherKey(result.CipherKey)) + require.NoError(t, err) + } + if result.Gzip > 0 { + recovered, err = util.DecompressData(recovered) + require.NoError(t, err) + } + require.Equal(t, plain, recovered) + }) + } +} + +func TestVolumeJwtExpiryRefusalBoundary(t *testing.T) { + key := security.SigningKey("disposable-volume-boundary") + for _, tc := range []struct { + name string + expired, nbf, iat, badKey, badFid, missing, urlencoded, suffix, noKey bool + want int + marker bool + }{ + {name: "expired", expired: true, want: 401, marker: true}, + {name: "expired_suffix", expired: true, suffix: true, want: 401, marker: true}, + {name: "expired_future_nbf", expired: true, nbf: true, want: 401}, + {name: "expired_future_iat", expired: true, iat: true, want: 401}, + {name: "expired_wrong_signature", expired: true, badKey: true, want: 401}, + {name: "expired_wrong_fid", expired: true, badFid: true, want: 401}, + {name: "missing", missing: true, want: 401}, + {name: "nonmultipart", expired: true, urlencoded: true, want: 401}, + {name: "valid", want: 201}, + {name: "no_key", missing: true, noKey: true, want: 201}, + } { + t.Run(tc.name, func(t *testing.T) { + vs := expiryVolume(t, key) + if tc.noKey { + vs.guard = security.NewGuard(nil, "", 0, "", 0) + } + plain := []byte("boundary content") + var body bytes.Buffer + mw := multipart.NewWriter(&body) + part, e := mw.CreateFormFile("file", "sample.bin") + require.NoError(t, e) + _, e = part.Write(plain) + require.NoError(t, e) + require.NoError(t, mw.Close()) + contentType := mw.FormDataContentType() + if tc.urlencoded { + body.Reset() + body.WriteString("sample=value") + contentType = "application/x-www-form-urlencoded" + } + counted := &expiryCountingBody{ReadCloser: io.NopCloser(bytes.NewReader(body.Bytes()))} + path := "/1,0000000100001234" + if tc.suffix { + path += "_2" + } + req := httptest.NewRequest(http.MethodPost, path, counted) + req.Header.Set("Content-Type", contentType) + // A request-supplied marker must never be echoed as response proof. + req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append") + if !tc.missing { + fid := "1,0000000100001234" + if tc.badFid { + fid = "1,0000000200001234" + } + exp := time.Now().Add(time.Hour) + if tc.expired { + exp = time.Now().Add(-time.Hour) + } + claims := security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(exp)}} + if tc.nbf { + claims.NotBefore = jwt.NewNumericDate(time.Now().Add(time.Hour)) + } + if tc.iat { + claims.IssuedAt = jwt.NewNumericDate(time.Now().Add(time.Hour)) + } + signingKey := key + if tc.badKey { + signingKey = security.SigningKey("different-disposable-key") + } + token, e := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey)) + require.NoError(t, e) + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + vs.PostHandler(rec, req) + require.Equal(t, tc.want, rec.Code) + want := "" + if tc.marker { + want = "expired-jwt-before-append" + } + require.Equal(t, want, rec.Header().Get("X-Seaweed-Write-Outcome")) + n := &needle.Needle{Id: 1, Cookie: 0x1234} + _, readErr := vs.store.ReadVolumeNeedle(1, n, nil, nil) + if tc.want == 401 { + require.Error(t, readErr) + if !tc.urlencoded { + require.Zero(t, counted.bytesRead) + } + } else { + require.NoError(t, readErr) + require.Equal(t, plain, n.Data) + } + }) + } +} + +func TestVolumeJwtExpiryProxyForwarding(t *testing.T) { + volumeKey := security.SigningKey("disposable-proxy-volume-key") + filerKey := security.SigningKey("disposable-proxy-filer-key") + vs := expiryVolume(t, volumeKey) + var hits atomic.Int32 + volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) == 1 { + claims := &security.SeaweedFileIdClaims{} + parsed, err := security.DecodeJwt(volumeKey, security.GetJwt(r), claims) + if err != nil || !parsed.Valid || claims.ExpiresAt == nil { + w.WriteHeader(500) + return + } + timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond))) + defer timer.Stop() + select { + case <-timer.C: + case <-r.Context().Done(): + return + } + } + vs.PostHandler(w, r) + })) + defer volume.Close() + fs := &FilerServer{filerGuard: security.NewGuard(nil, string(filerKey), 3600, "", 0), volumeGuard: security.NewGuard(nil, string(volumeKey), 10, "", 0)} + fid := "1,0000000100001234" + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Exercise the same real caller verifier as filerHandler, then its actual + // forwarding function with an already-resolved synthetic target. + if _, ok := fs.authenticateFilerJwt(r, true); !ok { + w.WriteHeader(401) + return + } + fs.proxyToVolumeServerURL(w, r, fid, volume.URL+"/"+fid) + })) + defer proxy.Close() + caller := string(security.GenJwtForFilerServer(filerKey, 3600)) + for i, want := range []int{401, 401, 201} { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + part, err := mw.CreateFormFile("file", "proxy.bin") + require.NoError(t, err) + _, err = part.Write([]byte("proxy clear data")) + require.NoError(t, err) + require.NoError(t, mw.Close()) + req, err := http.NewRequest(http.MethodPost, proxy.URL+"/?proxyChunkId="+fid, bytes.NewReader(body.Bytes())) + require.NoError(t, err) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("X-Seaweed-Write-Outcome", "expired-jwt-before-append") + token := caller + if i == 1 { + token = "invalid-disposable-caller" + } + req.Header.Set("Authorization", "Bearer "+token) + client := proxy.Client() + client.Timeout = 25 * time.Second + resp, err := client.Do(req) + require.NoError(t, err) + _, err = io.Copy(io.Discard, resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, want, resp.StatusCode) + marker := "" + if i == 0 { + marker = "expired-jwt-before-append" + } + require.Equal(t, marker, resp.Header.Get("X-Seaweed-Write-Outcome")) + if i == 1 { + require.Equal(t, int32(1), hits.Load()) + } + } + require.Equal(t, int32(2), hits.Load()) +} + +func TestVolumeJwtExpiryFanoutRollback(t *testing.T) { + key := security.SigningKey("disposable-fanout-expiry-key") + primary, replica := expiryVolume(t, key), expiryVolume(t, key) + firstCommitted := make(chan struct{}) + var committed sync.Once + var mu sync.Mutex + var firstReplicaStatuses []int + var firstReplicaMarkers []string + a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + primary.DeleteHandler(w, r) + return + } + rec := httptest.NewRecorder() + primary.PostHandler(rec, r) + if r.URL.Path == "/1,0000000100001234" && rec.Code == 201 { + committed.Do(func() { close(firstCommitted) }) + } + for k, v := range rec.Header() { + w.Header()[k] = v + } + w.WriteHeader(rec.Code) + _, _ = w.Write(rec.Body.Bytes()) + })) + defer a.Close() + b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + replica.DeleteHandler(w, r) + return + } + if r.URL.Path == "/1,0000000100001234" { + select { + case <-firstCommitted: + case <-r.Context().Done(): + return + } + claims := &security.SeaweedFileIdClaims{} + parsed, err := security.DecodeJwt(key, security.GetJwt(r), claims) + if err != nil || !parsed.Valid || claims.ExpiresAt == nil { + w.WriteHeader(500) + return + } + timer := time.NewTimer(time.Until(claims.ExpiresAt.Time.Add(100 * time.Millisecond))) + defer timer.Stop() + select { + case <-timer.C: + case <-r.Context().Done(): + return + } + } + rec := httptest.NewRecorder() + replica.PostHandler(rec, r) + if r.URL.Path == "/1,0000000100001234" { + mu.Lock() + firstReplicaStatuses = append(firstReplicaStatuses, rec.Code) + firstReplicaMarkers = append(firstReplicaMarkers, rec.Header().Get("X-Seaweed-Write-Outcome")) + mu.Unlock() + } + for k, v := range rec.Header() { + w.Header()[k] = v + } + w.WriteHeader(rec.Code) + _, _ = w.Write(rec.Body.Bytes()) + })) + defer b.Close() + var assignments int + plain := bytes.Repeat([]byte("fanout content"), 4096) + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + result, err := operation.UploadReaderInChunks(ctx, bytes.NewReader(plain), &operation.ChunkedUploadOption{ + ChunkSize: 1 << 20, MimeType: "application/octet-stream", AssignFunc: func(_ context.Context, _ int, _ uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) { + assignments++ + fid := "1,0000000100001234" + if assignments > 1 { + fid = "1,0000000200001234" + } + return nil, &operation.AssignResult{Fid: fid, Url: strings.TrimPrefix(a.URL, "http://"), Auth: security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(10*time.Second))), Replicas: []operation.Location{{Url: strings.TrimPrefix(b.URL, "http://")}}, Fsync: true}, nil + }, LookupJwtFunc: func(_ context.Context, fid string) security.EncodedJwt { + return security.EncodedJwt(expirySignedToken(t, key, fid, time.Now().Add(time.Hour))) + }, + }) + require.NoError(t, err) + require.Equal(t, 2, assignments) + mu.Lock() + gotStatuses := append([]int(nil), firstReplicaStatuses...) + gotMarkers := append([]string(nil), firstReplicaMarkers...) + mu.Unlock() + require.Equal(t, []int{401}, gotStatuses) + require.Equal(t, []string{"expired-jwt-before-append"}, gotMarkers) + require.Len(t, result.FileChunks, 1) + require.Equal(t, "1,0000000200001234", result.FileChunks[0].FileId) + for _, vs := range []*VolumeServer{primary, replica} { + _, err := vs.store.ReadVolumeNeedle(1, &needle.Needle{Id: 1, Cookie: 0x1234}, nil, nil) + require.Error(t, err) + n := &needle.Needle{Id: 2, Cookie: 0x1234} + _, err = vs.store.ReadVolumeNeedle(1, n, nil, nil) + require.NoError(t, err) + data := n.Data + if result.FileChunks[0].IsCompressed { + data, err = util.DecompressData(data) + require.NoError(t, err) + } + require.Equal(t, plain, data) + } +} diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index b5b92cf41f..f2abe020e9 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -330,19 +330,23 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req } func (vs *VolumeServer) maybeCheckJwtAuthorization(r *http.Request, vid, fid string, isWrite bool) bool { - var signingKey security.SigningKey - if isWrite { signingKey = vs.guard.SigningKey() - if len(signingKey) == 0 { - return true - } } else { signingKey = vs.guard.ReadSigningKey() - if len(signingKey) == 0 { - return true - } + } + return vs.checkJwtAuthorizationWithKey(r, vid, fid, signingKey) +} + +// checkJwtAuthorizationWithKey authorizes against a caller-snapshotted signing +// key, so a write handler can classify an expired-JWT refusal against the exact +// key it authorized under rather than re-reading one that may have rotated +// between the two reads. +func (vs *VolumeServer) checkJwtAuthorizationWithKey(r *http.Request, vid, fid string, signingKey security.SigningKey) bool { + + if len(signingKey) == 0 { + return true } tokenStr := security.GetJwt(r) diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index be2bef2db3..27d58db2b6 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "errors" "fmt" + "mime" "net/http" "strconv" "strings" @@ -14,11 +15,13 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/topology" "github.com/seaweedfs/seaweedfs/weed/util/buffer_pool" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var jpegStart = []byte{0xFF, 0xD8, 0xFF} @@ -52,7 +55,22 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { return } - if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) { + // Snapshot the write signing key once, so the authorization check and the + // expired-JWT classification below read the same key even if it rotates. + writeSigningKey := vs.guard.SigningKey() + if !vs.checkJwtAuthorizationWithKey(r, vid, fid, writeSigningKey) { + // ParseForm does not read multipart bodies. Preserve its existing placement + // and status precedence for other content types, which get no retry marker. + contentType, _, typeErr := mime.ParseMediaType(r.Header.Get("Content-Type")) + if typeErr == nil && contentType == "multipart/form-data" { + baseFid := fid + if sep := strings.LastIndex(baseFid, "_"); sep > 0 { + baseFid = baseFid[:sep] + } + if security.IsExpiredVolumeJwt(writeSigningKey, security.GetJwt(r), vid+","+baseFid) { + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeExpiredJwtBeforeAppend) + } + } writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) return } @@ -77,7 +95,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { status = http.StatusConflict } if errors.Is(writeError, topology.ErrorWriteRefused) { - w.Header().Set("X-Seaweed-Write-Outcome", "refused-before-append") + w.Header().Set(util_http.WriteOutcomeHeader, util_http.WriteOutcomeRefusedBeforeAppend) } writeJsonError(w, r, status, writeError) return diff --git a/weed/util/http/write_outcome.go b/weed/util/http/write_outcome.go new file mode 100644 index 0000000000..fd39c42522 --- /dev/null +++ b/weed/util/http/write_outcome.go @@ -0,0 +1,9 @@ +package http + +// Write outcomes describe a refusal of this HTTP attempt before append, not the +// absence of a stored needle from an earlier attempt with a lost response. +const ( + WriteOutcomeHeader = "X-Seaweed-Write-Outcome" + WriteOutcomeRefusedBeforeAppend = "refused-before-append" + WriteOutcomeExpiredJwtBeforeAppend = "expired-jwt-before-append" +)