package weed_server import ( "context" "net/http" "net/http/httptest" "net/url" "strings" "sync" "sync/atomic" "testing" "time" "github.com/seaweedfs/seaweedfs/weed/security" ) const ( proxyTestWriteKey = "cluster-write-key" proxyTestReadKey = "cluster-read-key" proxyTestVid = "3" proxyTestFid = "01637037d6" proxyTestFileId = proxyTestVid + "," + proxyTestFid ) // proxyTestVolume is a stand-in volume server that records what the filer // actually sent. Recording arrival separately from the header is what keeps the // negative assertions honest: an absent Authorization and a request that never // left the filer are otherwise indistinguishable. type proxyTestVolume struct { *httptest.Server hits atomic.Int32 auth atomic.Value // string effectiveJwt atomic.Value // string rawQuery atomic.Value // string } func newProxyTestVolume(t *testing.T) *proxyTestVolume { t.Helper() v := &proxyTestVolume{} v.auth.Store("") v.effectiveJwt.Store("") v.rawQuery.Store("") v.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { v.hits.Add(1) v.auth.Store(r.Header.Get("Authorization")) v.rawQuery.Store(r.URL.RawQuery) // The credential the volume server would actually evaluate, which is not // necessarily the Authorization header. v.effectiveJwt.Store(string(security.GetJwt(r))) })) t.Cleanup(func() { v.Close() // proxyToVolumeServerURL keys the semaphore map by host, and every // httptest server binds a fresh port; drop ours so -count=N runs do not // grow the map without bound. if u, err := url.Parse(v.URL); err == nil { proxySemaphores.Delete(u.Host) } }) return v } func (v *proxyTestVolume) seenAuth() string { return v.auth.Load().(string) } // seenEffectiveJwt is the token the volume server would validate, resolved the // same way VolumeServer.maybeCheckJwtAuthorization resolves it. func (v *proxyTestVolume) seenEffectiveJwt() string { return v.effectiveJwt.Load().(string) } func (v *proxyTestVolume) seenRawQuery() string { return v.rawQuery.Load().(string) } func (v *proxyTestVolume) requireReached(t *testing.T) { t.Helper() if v.hits.Load() == 0 { t.Fatal("request never reached the volume server, so the assertion below proves nothing") } } // security.GetJwt reads the "jwt" query parameter before the Authorization // header, so a caller-supplied one would outrank the token the filer attaches // on a read -- the credential the volume server evaluates has to be the filer's. func TestProxyReadDropsCallerJwtQueryParam(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodHead} { t.Run(method, func(t *testing.T) { volume := newProxyTestVolume(t) fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} r := httptest.NewRequest(method, "http://filer:8888/?proxyChunkId="+proxyTestFileId+"&jwt=caller-supplied&readDeleted=true", nil) fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) volume.requireReached(t) if got := volume.seenEffectiveJwt(); got == "caller-supplied" { t.Fatal("caller's jwt query param outranked the filer-minted token") } // The credential has to be a filer-minted read token for THIS file. // Comparing it against a separately minted token would also say that, // but only within the second that minted both: the expiry claim has // one-second resolution, so two mints either side of a tick differ in // the encoded string while carrying the same authority and file id. claims := &security.SeaweedFileIdClaims{} if _, err := security.DecodeJwt(security.SigningKey(proxyTestReadKey), security.EncodedJwt(volume.seenEffectiveJwt()), claims); err != nil { t.Fatalf("volume server would evaluate %q, which does not validate against the read key: %v", volume.seenEffectiveJwt(), err) } if claims.Fid != proxyTestFileId { t.Fatalf("token authorizes file %q, want %q", claims.Fid, proxyTestFileId) } if q := volume.seenRawQuery(); strings.Contains(q, "jwt=") { t.Fatalf("jwt survived in the forwarded query: %q", q) } // Unrelated params must still be forwarded. if q := volume.seenRawQuery(); !strings.Contains(q, "readDeleted=true") { t.Fatalf("readDeleted was dropped from the forwarded query: %q", q) } }) } } // A writer's credential is its own either way, so the query parameter is left // alone on writes -- stripping it would break a caller that presents its volume // JWT that way. func TestProxyWriteKeepsCallerJwtQueryParam(t *testing.T) { volume := newProxyTestVolume(t) fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} r := httptest.NewRequest(http.MethodPost, "http://filer:8888/?proxyChunkId="+proxyTestFileId+"&jwt=caller-supplied", nil) fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) volume.requireReached(t) if got := volume.seenEffectiveJwt(); got != "caller-supplied" { t.Fatalf("writer's own jwt query param was altered: got %q", got) } } // Everything the filer can hand a caller on the proxy path is reachable without // authentication, because the branch runs ahead of the filer's JWT gate. With // only a write key configured it must therefore mint nothing at all. func TestProxyMintsNothingWithoutReadKey(t *testing.T) { fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, "", 10)} if jwt := fs.maybeGetVolumeReadJwtAuthorizationToken(proxyTestFileId); jwt != "" { t.Fatalf("minted %q with no read key configured", jwt) } } // A configured read key still yields a read token, and it stays read-only. func TestProxyReadTokenIsReadOnly(t *testing.T) { fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} jwt := fs.maybeGetVolumeReadJwtAuthorizationToken(proxyTestFileId) if jwt == "" { t.Fatal("no read token minted despite a configured read key") } vs := &VolumeServer{guard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} read := httptest.NewRequest(http.MethodGet, "http://volume:8080/"+proxyTestFileId, nil) read.Header.Set("Authorization", security.BearerPrefix+jwt) if !vs.maybeCheckJwtAuthorization(read, proxyTestVid, proxyTestFid, false) { t.Fatal("read token rejected on a read") } write := httptest.NewRequest(http.MethodDelete, "http://volume:8080/"+proxyTestFileId, nil) write.Header.Set("Authorization", security.BearerPrefix+jwt) if vs.maybeCheckJwtAuthorization(write, proxyTestVid, proxyTestFid, true) { t.Fatal("read token authorized a write") } } // Writes must reach the volume server carrying the caller's own AssignVolume // token and nothing else. POST is the method every in-tree proxied uploader // actually sends, so it leads the table. func TestProxyWriteCarriesOnlyCallerCredential(t *testing.T) { callerToken := security.BearerPrefix + string(security.GenJwtForVolumeServer(security.SigningKey(proxyTestWriteKey), 10, proxyTestFileId)) for _, tc := range []struct { name string method string readKey string sent string want string }{ {"anonymous post", http.MethodPost, "", "", ""}, {"anonymous post with read key", http.MethodPost, proxyTestReadKey, "", ""}, {"anonymous delete", http.MethodDelete, "", "", ""}, {"anonymous delete with read key", http.MethodDelete, proxyTestReadKey, "", ""}, {"anonymous put with read key", http.MethodPut, proxyTestReadKey, "", ""}, {"caller token forwarded on post", http.MethodPost, proxyTestReadKey, callerToken, callerToken}, {"caller token forwarded on delete", http.MethodDelete, "", callerToken, callerToken}, } { t.Run(tc.name, func(t *testing.T) { volume := newProxyTestVolume(t) fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, tc.readKey, 10)} r := httptest.NewRequest(tc.method, "http://filer:8888/?proxyChunkId="+proxyTestFileId, nil) if tc.sent != "" { r.Header.Set("Authorization", tc.sent) } fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) volume.requireReached(t) if got := volume.seenAuth(); got != tc.want { t.Fatalf("volume server saw Authorization %q, want %q", got, tc.want) } }) } } // Reads keep the minted token so weed mount can read through the proxy against a // volume server that enforces read JWTs -- and the minted token must *replace* // whatever the caller sent, not be appended alongside it. func TestProxyReadReplacesCallerCredential(t *testing.T) { volume := newProxyTestVolume(t) fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} r := httptest.NewRequest(http.MethodGet, "http://filer:8888/?proxyChunkId="+proxyTestFileId, nil) r.Header.Set("Authorization", security.BearerPrefix+"caller-supplied-token") fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) volume.requireReached(t) seen := volume.seenAuth() if seen == security.BearerPrefix+"caller-supplied-token" { t.Fatal("caller's token reached the volume server instead of the minted one") } vs := &VolumeServer{guard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} check := httptest.NewRequest(http.MethodGet, "http://volume:8080/"+proxyTestFileId, nil) check.Header.Set("Authorization", seen) if !vs.maybeCheckJwtAuthorization(check, proxyTestVid, proxyTestFid, false) { t.Fatalf("forwarded token %q did not authorize the read", seen) } } // With no read key there is nothing to mint, and the caller's Authorization on // the read path is a filer credential -- it must be dropped, not relayed to a // volume server that has no business seeing it. func TestProxyReadDropsCallerCredentialWhenNothingMinted(t *testing.T) { for _, method := range []string{http.MethodGet, http.MethodHead} { t.Run(method, func(t *testing.T) { volume := newProxyTestVolume(t) fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, "", 10)} r := httptest.NewRequest(method, "http://filer:8888/?proxyChunkId="+proxyTestFileId, nil) r.Header.Set("Authorization", security.BearerPrefix+"filer-credential") fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) volume.requireReached(t) if got := volume.seenAuth(); got != "" { t.Fatalf("volume server saw Authorization %q, want it dropped", got) } }) } } // Writes must not queue behind the read semaphore: a proxied write carries an // AssignVolume token that expires 10s after the assign by default, and waiting // for a read slot can push it past expiry. func TestProxyWriteBypassesReadSemaphore(t *testing.T) { volume := newProxyTestVolume(t) host := volume.Listener.Addr().String() // Fill every read slot for this host and never release them. for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { if err := acquireProxySemaphore(context.Background(), host); err != nil { t.Fatalf("fill slot %d: %v", i, err) } } defer func() { for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { releaseProxySemaphore(host) } }() fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, "", 10)} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() r := httptest.NewRequest(http.MethodPost, "http://filer:8888/?proxyChunkId="+proxyTestFileId, nil).WithContext(ctx) done := make(chan struct{}) go func() { defer close(done) fs.proxyToVolumeServerURL(httptest.NewRecorder(), r, proxyTestFileId, volume.URL+"/"+proxyTestFileId) }() select { case <-done: case <-time.After(3 * time.Second): t.Fatal("proxied write blocked on the read semaphore") } volume.requireReached(t) } // The volume server strips a _N delta suffix before comparing the fid claim, so // a token minted for the suffixed form would never validate. func TestProxyReadTokenMatchesDeltaFid(t *testing.T) { const deltaFileId = proxyTestFileId + "_1" fs := &FilerServer{volumeGuard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} jwt := fs.maybeGetVolumeReadJwtAuthorizationToken(deltaFileId) if jwt == "" { t.Fatal("no read token minted for a delta fid") } vs := &VolumeServer{guard: security.NewGuard([]string{}, proxyTestWriteKey, 10, proxyTestReadKey, 10)} r := httptest.NewRequest(http.MethodGet, "http://volume:8080/"+deltaFileId, nil) r.Header.Set("Authorization", security.BearerPrefix+jwt) if !vs.maybeCheckJwtAuthorization(r, proxyTestVid, proxyTestFid+"_1", false) { t.Fatal("token minted for a delta fid did not authorize the read") } } func TestValidateProxyChunkId(t *testing.T) { for _, tc := range []struct { fileId string ok bool }{ {"3,01637037d6", true}, {"1,0c2b3f2f0f", true}, {"12,04f0e6ba1d", true}, {"3,01637037d6_1", true}, // batch-assign delta form {"3,01637037d6_12", true}, // multi-digit delta {"3,x/../../status", false}, {"3,01637037d6/../../status", false}, {"3,01637037d6/../../stats/counter", false}, {"3,../../status", false}, {"3,01637037d6/../../status_1", false}, // traversal wearing a delta suffix // The suffix must be digits only, or stripping it would reduce a // traversal payload to a valid fid and let it through. {"3,01637037d6_1/../../status", false}, {"3,01637037d6_../../status", false}, {"3,01637037d6_1/../../stats/counter", false}, {"3,01637037d6_", false}, {"3,01637037d6_abc", false}, {"3,01637037d6_1a", false}, {"3,01637037d6?readDeleted=true", false}, {"3,01637037d6#frag", false}, {"3,", false}, {"3,abc", false}, {"3", false}, {"", false}, } { err := validateProxyChunkId(tc.fileId) if tc.ok && err != nil { t.Errorf("validateProxyChunkId(%q) rejected a valid fid: %v", tc.fileId, err) } if !tc.ok && err == nil { t.Errorf("validateProxyChunkId(%q) accepted a malformed fid", tc.fileId) } } } // A fid carrying dot segments must be rejected before the lookup, so it can // never be pasted into a volume server URL. Asserting on 400 (not merely "no // traversal") also proves the request never left the filer. func TestProxyRejectsTraversalBeforeLookup(t *testing.T) { fs := &FilerServer{} for _, fileId := range []string{ "3,x/../../status", "3,01637037d6/../../status", "3,01637037d6/../../stats/counter", "3,01637037d6_1/../../status", "3,01637037d6_../../status", } { r := httptest.NewRequest(http.MethodGet, "http://filer:8888/?proxyChunkId="+fileId, nil) w := httptest.NewRecorder() // fs.filer is nil: reaching the lookup would panic, so surviving this // call is itself proof the fid was rejected first. fs.proxyToVolumeServer(w, r, fileId) if w.Code != http.StatusBadRequest { t.Errorf("proxyChunkId=%q returned %d, want 400", fileId, w.Code) } } } func TestProxySemaphore_LimitsConcurrency(t *testing.T) { host := "test-volume:8080" defer proxySemaphores.Delete(host) var running atomic.Int32 var maxSeen atomic.Int32 var wg sync.WaitGroup // Launch more goroutines than the semaphore allows total := proxyReadConcurrencyPerVolumeServer * 3 for i := 0; i < total; i++ { wg.Add(1) go func() { defer wg.Done() if err := acquireProxySemaphore(context.Background(), host); err != nil { t.Errorf("acquire: %v", err) return } defer releaseProxySemaphore(host) cur := running.Add(1) // Track peak concurrency for { old := maxSeen.Load() if cur <= old || maxSeen.CompareAndSwap(old, cur) { break } } time.Sleep(time.Millisecond) running.Add(-1) }() } wg.Wait() peak := maxSeen.Load() if peak > int32(proxyReadConcurrencyPerVolumeServer) { t.Fatalf("peak concurrency %d exceeded limit %d", peak, proxyReadConcurrencyPerVolumeServer) } if peak == 0 { t.Fatal("no goroutines ran") } } func TestProxySemaphore_ContextCancellation(t *testing.T) { host := "test-cancel:8080" defer proxySemaphores.Delete(host) // Fill the semaphore for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { if err := acquireProxySemaphore(context.Background(), host); err != nil { t.Fatalf("fill acquire: %v", err) } } // Try to acquire with a cancelled context ctx, cancel := context.WithCancel(context.Background()) cancel() err := acquireProxySemaphore(ctx, host) if err == nil { t.Fatal("expected error from cancelled context") } // Clean up for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { releaseProxySemaphore(host) } } func TestProxySemaphore_PerHostIsolation(t *testing.T) { hostA := "volume-a:8080" hostB := "volume-b:8080" defer proxySemaphores.Delete(hostA) defer proxySemaphores.Delete(hostB) // Fill hostA's semaphore for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { if err := acquireProxySemaphore(context.Background(), hostA); err != nil { t.Fatalf("fill hostA: %v", err) } } // hostB should still be acquirable if err := acquireProxySemaphore(context.Background(), hostB); err != nil { t.Fatalf("hostB should not be blocked by hostA: %v", err) } releaseProxySemaphore(hostB) // Clean up hostA for i := 0; i < proxyReadConcurrencyPerVolumeServer; i++ { releaseProxySemaphore(hostA) } }