vector: bound request buffering before authentication and document installation (#12537)

* s3vectors: bound body buffering before authentication

* docs: include vector engine installation and source builds

* s3vectors: prevent MIME sniffing of JSON error responses

* docs: load vector Cargo configuration when building from source

* docs: note the native protoc requirement for Windows vector builds

* docs: make vector source checkout and working directory explicit

* ci: handle deb822 sources during Ubuntu mirror failover

* ci: bound Ubuntu mirror attempts and reject partial index updates
This commit is contained in:
Chris Lu authored and GitHub committed 2026-11-10 04:21:56 -08:00
1 parent 8236c8d136
commit 6c473433c5
9 files changed
+308 -20

No files matched your search

+13
View File
@@ -10,12 +10,25 @@ import (
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"golang.org/x/sync/semaphore"
)
func (s *Server) SetLimiter(limit func(http.HandlerFunc, string) http.HandlerFunc) {
s.limit = limit
}
func (s *Server) SetRequestLimits(maxRequests, maxBytes int64) {
if maxRequests <= 0 {
maxRequests = 16
}
if maxBytes <= 0 {
maxBytes = 8 * maxRequestBodyBytes
}
s.requestSlots = semaphore.NewWeighted(maxRequests)
s.requestBytes = semaphore.NewWeighted(maxBytes)
s.requestByteLimit = maxBytes
}
func (s *Server) limitRequest(handler http.HandlerFunc, w http.ResponseWriter, r *http.Request) {
op := operationFromRequest(r)
+117
View File
@@ -2,10 +2,16 @@ package s3vectors
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -123,3 +129,114 @@ func TestVectorLimitsCountChunkedBodyBytes(t *testing.T) {
t.Fatal("mutated original request")
}
}
type admissionAuth struct {
started, release chan struct{}
calls atomic.Int32
}
func (a *admissionAuth) DefaultAllow() bool { return true }
func (a *admissionAuth) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) {
a.calls.Add(1)
_, _ = io.ReadAll(r.Body)
if r.Header.Get("Block") != "" {
close(a.started)
select {
case <-a.release:
case <-r.Context().Done():
}
}
if r.Header.Get("Deny") != "" {
return "", nil, s3err.ErrSignatureDoesNotMatch
}
return "tester", nil, s3err.ErrNone
}
func TestVectorAdmissionBoundsBufferingBeforeAuthentication(t *testing.T) {
for _, tc := range []struct {
name string
requests, bytes int64
chunked bool
}{
{"request count", 1, 10, false},
{"byte count", 2, 2, false},
{"chunked body", 2, 2, true},
} {
t.Run(tc.name, func(t *testing.T) {
auth := &admissionAuth{started: make(chan struct{}), release: make(chan struct{})}
server := NewServer(nil, auth, nil)
server.SetRequestLimits(tc.requests, tc.bytes)
handler := server.Auth(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
request := func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/PutVectors", strings.NewReader("{}"))
if tc.chunked {
r.ContentLength = -1
}
return r
}
first := request()
first.Header.Set("Block", "true")
firstCtx, cancelFirst := context.WithTimeout(t.Context(), 5*time.Second)
defer cancelFirst()
done := make(chan struct{})
go func() { defer close(done); handler(httptest.NewRecorder(), first.WithContext(firstCtx)) }()
select {
case <-auth.started:
case <-firstCtx.Done():
t.Fatal("first request did not enter authentication")
}
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
waiting := httptest.NewRecorder()
handler(waiting, request().WithContext(ctx))
if waiting.Code != http.StatusRequestTimeout || auth.calls.Load() != 1 {
t.Fatalf("waiting request buffered/authenticated: status %d, calls %d", waiting.Code, auth.calls.Load())
}
close(auth.release)
<-done
denied := request()
denied.Header.Set("Deny", "true")
handler(httptest.NewRecorder(), denied.WithContext(firstCtx))
after := httptest.NewRecorder()
handler(after, request().WithContext(firstCtx))
if after.Code != http.StatusNoContent || auth.calls.Load() != 3 {
t.Fatalf("capacity leaked after success/failure: status %d, calls %d", after.Code, auth.calls.Load())
}
})
}
}
func TestVectorErrorResponsesEscapeHTML(t *testing.T) {
payload := "<script>alert(1)</script>"
for _, limited := range []bool{false, true} {
server := NewServer(nil, headerAuth{}, nil)
operation := payload
body := `{}`
if limited {
operation = "QueryVectors"
body = `{"vectorBucketName":"embeddings","indexName":"documents"}`
server.SetLimiter(func(http.HandlerFunc, string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTooManyRequests)
_ = xml.NewEncoder(w).Encode(struct {
XMLName xml.Name `xml:"Error"`
Code, Message string
}{Code: "SlowDown", Message: payload})
}
})
}
router := mux.NewRouter()
server.RegisterRoutes(router)
request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
request.Header.Set("X-Amz-Target", operation)
response := httptest.NewRecorder()
router.ServeHTTP(response, request)
var failure Error
if response.Code < 400 || json.Unmarshal(response.Body.Bytes(), &failure) != nil || !strings.Contains(failure.Message, payload) {
t.Fatalf("error response lost message: %d %s", response.Code, response.Body)
}
if strings.Contains(response.Body.String(), "<") || response.Header().Get("X-Content-Type-Options") != "nosniff" || response.Header().Get("Content-Type") != "application/x-amz-json-1.1" {
t.Fatalf("unsafe error response: %v %s", response.Header(), response.Body)
}
}
}
+36 -9
View File
@@ -17,6 +17,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"golang.org/x/sync/semaphore"
)
// FilerClient provides access to the filer for storage operations.
@@ -32,14 +33,17 @@ type S3Authenticator interface {
// Server implements the AWS S3 Vectors API.
type Server struct {
filerClient FilerClient
tables *s3tables.Manager
internal *s3tables.Manager
vectors vector_pb.VectorServiceClient
authenticator S3Authenticator
region string
accountID string
limit func(http.HandlerFunc, string) http.HandlerFunc
filerClient FilerClient
tables *s3tables.Manager
internal *s3tables.Manager
vectors vector_pb.VectorServiceClient
authenticator S3Authenticator
region string
accountID string
limit func(http.HandlerFunc, string) http.HandlerFunc
requestSlots *semaphore.Weighted
requestBytes *semaphore.Weighted
requestByteLimit int64
}
// NewServer creates an S3 Vectors server over the given filer and catalog.
@@ -53,13 +57,15 @@ func NewServer(filerClient FilerClient, authenticator S3Authenticator, vectors v
// demand read grants the API does not require.
internal := s3tables.NewManager()
internal.SetTrusted(true)
return &Server{
server := &Server{
filerClient: filerClient,
tables: manager,
internal: internal,
vectors: vectors,
authenticator: authenticator,
}
server.SetRequestLimits(0, 0)
return server
}
// SetRegion configures the region embedded in the ARNs this server mints.
@@ -158,11 +164,32 @@ func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) {
// Auth authenticates the caller and puts the identity in the request context.
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
if s.requestSlots == nil {
s.SetRequestLimits(0, 0)
}
return func(w http.ResponseWriter, r *http.Request) {
if r.ContentLength > maxRequestBodyBytes {
writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error())
return
}
if err := s.requestSlots.Acquire(r.Context(), 1); err != nil {
writeError(w, http.StatusRequestTimeout, "RequestTimeout", err.Error())
return
}
defer s.requestSlots.Release(1)
var size int64
if r.Body != nil && r.Body != http.NoBody {
size = r.ContentLength
if size <= 0 {
size = maxRequestBodyBytes
}
}
size = min(size, s.requestByteLimit)
if err := s.requestBytes.Acquire(r.Context(), size); err != nil {
writeError(w, http.StatusRequestTimeout, "RequestTimeout", err.Error())
return
}
defer s.requestBytes.Release(size)
r = r.WithContext(s3_constants.SetRequestBodyLimitInContext(r.Context(), maxRequestBodyBytes))
identity, identityObj, errCode := s.authenticator.AuthenticateRequest(r)
switch errCode {
+1
View File
@@ -22,6 +22,7 @@ func identityOf(r *http.Request) string {
func writeJSON(w http.ResponseWriter, status int, payload interface{}) {
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(status)
if payload != nil {
_ = json.NewEncoder(w).Encode(payload)