mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-10 16:40:46 +02:00
* s3: stop warming every bucket's config at startup Listing all buckets in BucketRegistry.init() made S3 gateway startup O(buckets) and pinned every bucket's metadata and config resident, which does not scale past a few hundred thousand buckets. Both caches already have lazy miss paths, so load on first access instead and let the metadata subscription refresh only entries already resident; cold buckets cost one filer round-trip on their first request. * s3: bound the per-bucket caches with LRU eviction The bucket config cache, bucket registry, and their negative caches were plain maps that only ever grew: the config cache TTL made Get miss but never evicted the entry, and the not-found sets grew on every probe of a nonexistent bucket name. Cap all four at 65536 entries with LRU eviction so a gateway keeps its hot working set and evicted buckets reload from the filer on next access. * s3: cache parsed bucket config instead of the full filer entry Each cached BucketConfig retained the whole bucket entry (extended attribute map plus raw content bytes) alongside the fields parsed from it, roughly doubling per-bucket cache cost and keeping data the read path never looks at. Parse everything up front in newBucketConfigFromEntry - now also the creator identity, tags, encryption config, and stored lifecycle XML - and drop the entry. updateBucketConfig now reads the entry fresh from the filer and diffs the mapped extended attributes against it, so the patch is computed against current state instead of a cached copy; the config clone helpers that existed for that path go away. * s3: dedup cold bucket-registry loads per bucket The registry's notFound lock doubled as the load serializer, holding one global mutex across the filer round-trip so first-touch requests for different buckets queued behind each other; the cache fill also happened after the lock was released, so two concurrent misses for the same bucket could both reach the filer. Replace it with a singleflight per bucket that fills the cache inside the flight: different buckets load concurrently, the same bucket loads once.
686 lines
22 KiB
Go
686 lines
22 KiB
Go
package s3api
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gorilla/mux"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
// TestPostPolicyKeyNormalization tests that object keys from presigned POST
|
|
// are properly normalized without leading slashes and with duplicate slashes removed.
|
|
// This ensures consistent key handling across the S3 API.
|
|
func TestPostPolicyKeyNormalization(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
key string
|
|
expectedObject string // Expected normalized object key
|
|
}{
|
|
{
|
|
name: "key without leading slash",
|
|
key: "test_image.png",
|
|
expectedObject: "test_image.png",
|
|
},
|
|
{
|
|
name: "key with leading slash",
|
|
key: "/test_image.png",
|
|
expectedObject: "test_image.png",
|
|
},
|
|
{
|
|
name: "key with path without leading slash",
|
|
key: "folder/subfolder/test_image.png",
|
|
expectedObject: "folder/subfolder/test_image.png",
|
|
},
|
|
{
|
|
name: "key with path with leading slash",
|
|
key: "/folder/subfolder/test_image.png",
|
|
expectedObject: "folder/subfolder/test_image.png",
|
|
},
|
|
{
|
|
name: "simple filename",
|
|
key: "file.txt",
|
|
expectedObject: "file.txt",
|
|
},
|
|
{
|
|
name: "key with duplicate slashes",
|
|
key: "folder//subfolder///file.txt",
|
|
expectedObject: "folder/subfolder/file.txt",
|
|
},
|
|
{
|
|
name: "key with leading duplicate slashes",
|
|
key: "//folder/file.txt",
|
|
expectedObject: "folder/file.txt",
|
|
},
|
|
{
|
|
name: "key with trailing slash",
|
|
key: "folder/",
|
|
expectedObject: "folder/",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Use the actual NormalizeObjectKey function
|
|
object := s3_constants.NormalizeObjectKey(tt.key)
|
|
|
|
// Verify the normalized object matches expected
|
|
assert.Equal(t, tt.expectedObject, object,
|
|
"Key should be normalized correctly")
|
|
|
|
// Verify path construction would be correct
|
|
bucket := "my-bucket"
|
|
bucketsPath := "/buckets"
|
|
expectedPath := bucketsPath + "/" + bucket + "/" + tt.expectedObject
|
|
actualPath := bucketsPath + "/" + bucket + "/" + object
|
|
|
|
assert.Equal(t, expectedPath, actualPath,
|
|
"File path should be correctly constructed with slash between bucket and key")
|
|
|
|
// Verify we don't have double slashes (except at the start which is fine)
|
|
assert.NotContains(t, actualPath[1:], "//",
|
|
"Path should not contain double slashes")
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestNormalizeObjectKey tests the NormalizeObjectKey function directly
|
|
func TestNormalizeObjectKey(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
expected string
|
|
}{
|
|
{"empty string", "", ""},
|
|
{"simple file", "file.txt", "file.txt"},
|
|
{"with leading slash", "/file.txt", "file.txt"},
|
|
{"path without slash", "a/b/c.txt", "a/b/c.txt"},
|
|
{"path with slash", "/a/b/c.txt", "a/b/c.txt"},
|
|
{"duplicate slashes", "a//b///c.txt", "a/b/c.txt"},
|
|
{"leading duplicates", "///a/b.txt", "a/b.txt"},
|
|
{"all duplicates", "//a//b//", "a/b/"},
|
|
{"just slashes", "///", ""},
|
|
{"trailing slash", "folder/", "folder/"},
|
|
{"backslash to forward slash", "folder\\file.txt", "folder/file.txt"},
|
|
{"windows path", "folder\\subfolder\\file.txt", "folder/subfolder/file.txt"},
|
|
{"mixed slashes", "a/b\\c/d", "a/b/c/d"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
result := s3_constants.NormalizeObjectKey(tt.input)
|
|
assert.Equal(t, tt.expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestPostPolicyFilenameSubstitution tests the ${filename} substitution in keys
|
|
func TestPostPolicyFilenameSubstitution(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
keyTemplate string
|
|
uploadedFilename string
|
|
expectedKey string
|
|
}{
|
|
{
|
|
name: "filename at end",
|
|
keyTemplate: "uploads/${filename}",
|
|
uploadedFilename: "photo.jpg",
|
|
expectedKey: "uploads/photo.jpg",
|
|
},
|
|
{
|
|
name: "filename in middle",
|
|
keyTemplate: "user/files/${filename}/original",
|
|
uploadedFilename: "document.pdf",
|
|
expectedKey: "user/files/document.pdf/original",
|
|
},
|
|
{
|
|
name: "no substitution needed",
|
|
keyTemplate: "static/file.txt",
|
|
uploadedFilename: "ignored.txt",
|
|
expectedKey: "static/file.txt",
|
|
},
|
|
{
|
|
name: "filename only",
|
|
keyTemplate: "${filename}",
|
|
uploadedFilename: "myfile.png",
|
|
expectedKey: "myfile.png",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Simulate the substitution logic from PostPolicyBucketHandler
|
|
key := tt.keyTemplate
|
|
if tt.uploadedFilename != "" && strings.Contains(key, "${filename}") {
|
|
key = strings.Replace(key, "${filename}", tt.uploadedFilename, -1)
|
|
}
|
|
|
|
// Normalize using the actual function
|
|
object := s3_constants.NormalizeObjectKey(key)
|
|
|
|
assert.Equal(t, tt.expectedKey, object,
|
|
"Key should be correctly substituted and normalized")
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestExtractPostPolicyFormValues tests the form value extraction
|
|
func TestExtractPostPolicyFormValues(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
key string
|
|
contentType string
|
|
fileContent string
|
|
fileName string
|
|
expectSuccess bool
|
|
}{
|
|
{
|
|
name: "basic upload",
|
|
key: "test.txt",
|
|
contentType: "text/plain",
|
|
fileContent: "hello world",
|
|
fileName: "upload.txt",
|
|
expectSuccess: true,
|
|
},
|
|
{
|
|
name: "upload with path key",
|
|
key: "folder/subfolder/test.txt",
|
|
contentType: "application/octet-stream",
|
|
fileContent: "binary data",
|
|
fileName: "data.bin",
|
|
expectSuccess: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Create multipart form
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
|
|
// Add form fields
|
|
writer.WriteField("key", tt.key)
|
|
writer.WriteField("Content-Type", tt.contentType)
|
|
|
|
// Add file
|
|
part, err := writer.CreateFormFile("file", tt.fileName)
|
|
assert.NoError(t, err)
|
|
_, err = io.WriteString(part, tt.fileContent)
|
|
assert.NoError(t, err)
|
|
|
|
err = writer.Close()
|
|
assert.NoError(t, err)
|
|
|
|
// Parse the form
|
|
reader := multipart.NewReader(&buf, writer.Boundary())
|
|
form, err := reader.ReadForm(5 * 1024 * 1024)
|
|
assert.NoError(t, err)
|
|
defer form.RemoveAll()
|
|
|
|
// Extract values using the actual function
|
|
filePart, fileName, fileContentType, fileSize, formValues, err := extractPostPolicyFormValues(form)
|
|
|
|
if tt.expectSuccess {
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, filePart)
|
|
assert.Equal(t, tt.fileName, fileName)
|
|
assert.NotEmpty(t, fileContentType)
|
|
assert.Greater(t, fileSize, int64(0))
|
|
assert.Equal(t, tt.key, formValues.Get("Key"))
|
|
|
|
filePart.Close()
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestPostPolicyPathConstruction is an integration-style test that verifies
|
|
// the complete path construction logic
|
|
func TestPostPolicyPathConstruction(t *testing.T) {
|
|
s3a := &S3ApiServer{
|
|
option: &S3ApiServerOption{
|
|
BucketsPath: "/buckets",
|
|
},
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
bucket string
|
|
formKey string // Key as it would come from form (may not have leading slash)
|
|
expectedPath string
|
|
}{
|
|
{
|
|
name: "simple key without slash - the bug case",
|
|
bucket: "my-bucket",
|
|
formKey: "test_image.png",
|
|
expectedPath: "/buckets/my-bucket/test_image.png",
|
|
},
|
|
{
|
|
name: "simple key with slash",
|
|
bucket: "my-bucket",
|
|
formKey: "/test_image.png",
|
|
expectedPath: "/buckets/my-bucket/test_image.png",
|
|
},
|
|
{
|
|
name: "nested path without leading slash",
|
|
bucket: "uploads",
|
|
formKey: "2024/01/photo.jpg",
|
|
expectedPath: "/buckets/uploads/2024/01/photo.jpg",
|
|
},
|
|
{
|
|
name: "nested path with leading slash",
|
|
bucket: "uploads",
|
|
formKey: "/2024/01/photo.jpg",
|
|
expectedPath: "/buckets/uploads/2024/01/photo.jpg",
|
|
},
|
|
{
|
|
name: "key with duplicate slashes",
|
|
bucket: "my-bucket",
|
|
formKey: "folder//file.txt",
|
|
expectedPath: "/buckets/my-bucket/folder/file.txt",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Use the actual NormalizeObjectKey function
|
|
object := s3_constants.NormalizeObjectKey(tt.formKey)
|
|
|
|
// Construct path as done in PostPolicyBucketHandler
|
|
filePath := s3a.option.BucketsPath + "/" + tt.bucket + "/" + object
|
|
|
|
assert.Equal(t, tt.expectedPath, filePath,
|
|
"File path should be correctly constructed")
|
|
|
|
// Verify bucket and key are properly separated
|
|
assert.Contains(t, filePath, tt.bucket+"/",
|
|
"Bucket should be followed by a slash")
|
|
})
|
|
}
|
|
}
|
|
|
|
// canonicalFormValues builds an http.Header mirroring how
|
|
// extractPostPolicyFormValues canonicalizes the keys in the POST form.
|
|
func canonicalFormValues(pairs map[string]string) http.Header {
|
|
h := make(http.Header)
|
|
for k, v := range pairs {
|
|
h[http.CanonicalHeaderKey(k)] = []string{v}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_ForwardsAcl ensures the acl form field is
|
|
// promoted to the X-Amz-Acl header on the underlying PUT.
|
|
func TestApplyPostPolicyFormHeaders_ForwardsAcl(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
formValues := canonicalFormValues(map[string]string{
|
|
"acl": "public-read",
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
assert.Equal(t, "public-read", r.Header.Get(s3_constants.AmzCannedAcl))
|
|
assert.Equal(t, "public-read", r.Header.Get("X-Amz-Acl"))
|
|
// The raw "Acl" form key must not be copied verbatim onto the request.
|
|
assert.Empty(t, r.Header.Get("Acl"))
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_ForwardsContentHeaders ensures the
|
|
// Content-Encoding and Content-Language form fields are copied to the
|
|
// request header so they reach the underlying PUT.
|
|
func TestApplyPostPolicyFormHeaders_ForwardsContentHeaders(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
header string
|
|
value string
|
|
}{
|
|
{
|
|
name: "Content-Encoding",
|
|
header: "Content-Encoding",
|
|
value: "gzip",
|
|
},
|
|
{
|
|
name: "Content-Language",
|
|
header: "Content-Language",
|
|
value: "en-US",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
formValues := canonicalFormValues(map[string]string{
|
|
tt.header: tt.value,
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
assert.Equal(t, tt.value, r.Header.Get(tt.header))
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_ForwardsXAmzHeaders ensures arbitrary
|
|
// x-amz-* form fields (storage class, tagging, SSE, object lock, website
|
|
// redirect, metadata) are all forwarded to the request.
|
|
func TestApplyPostPolicyFormHeaders_ForwardsXAmzHeaders(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
formValues := canonicalFormValues(map[string]string{
|
|
"x-amz-storage-class": "STANDARD_IA",
|
|
"x-amz-tagging": "project=alpha&env=prod",
|
|
"x-amz-server-side-encryption": "AES256",
|
|
"x-amz-object-lock-mode": "GOVERNANCE",
|
|
"x-amz-website-redirect-location": "/redirected",
|
|
"x-amz-meta-foo": "bar",
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
assert.Equal(t, "STANDARD_IA", r.Header.Get("X-Amz-Storage-Class"))
|
|
assert.Equal(t, "project=alpha&env=prod", r.Header.Get("X-Amz-Tagging"))
|
|
assert.Equal(t, "AES256", r.Header.Get("X-Amz-Server-Side-Encryption"))
|
|
assert.Equal(t, "GOVERNANCE", r.Header.Get("X-Amz-Object-Lock-Mode"))
|
|
assert.Equal(t, "/redirected", r.Header.Get("X-Amz-Website-Redirect-Location"))
|
|
assert.Equal(t, "bar", r.Header.Get("X-Amz-Meta-Foo"))
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_SkipsReserved ensures POST-policy
|
|
// mechanism fields (signatures, key, bucket, success actions, etc.) are not
|
|
// leaked as headers on the forwarded PUT.
|
|
func TestApplyPostPolicyFormHeaders_SkipsReserved(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
formValues := canonicalFormValues(map[string]string{
|
|
"Policy": "base64-encoded-policy",
|
|
"Signature": "v2-signature",
|
|
"AWSAccessKeyId": "AKIAEXAMPLE",
|
|
"x-amz-signature": "v4-signature",
|
|
"x-amz-credential": "AKIAEXAMPLE/20260417/us-east-1/s3/aws4_request",
|
|
"x-amz-algorithm": "AWS4-HMAC-SHA256",
|
|
"x-amz-date": "20260417T000000Z",
|
|
"x-amz-security-token": "session-token",
|
|
"key": "uploads/file.txt",
|
|
"file": "ignored",
|
|
"bucket": "my-bucket",
|
|
"success_action_redirect": "https://example.com/ok",
|
|
"success_action_status": "201",
|
|
"redirect": "https://example.com/legacy",
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
reserved := []string{
|
|
"Policy",
|
|
"Signature",
|
|
"Awsaccesskeyid",
|
|
"X-Amz-Signature",
|
|
"X-Amz-Credential",
|
|
"X-Amz-Algorithm",
|
|
"X-Amz-Date",
|
|
"X-Amz-Security-Token",
|
|
"Key",
|
|
"File",
|
|
"Bucket",
|
|
"Success_action_redirect",
|
|
"Success_action_status",
|
|
"Redirect",
|
|
}
|
|
for _, k := range reserved {
|
|
assert.Empty(t, r.Header.Get(k), "reserved field %q must not be forwarded", k)
|
|
}
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_KeepsExistingCacheControl is a regression
|
|
// check that the headers previously forwarded (Cache-Control, Expires,
|
|
// Content-Disposition) remain forwarded by the refactored helper.
|
|
func TestApplyPostPolicyFormHeaders_KeepsExistingCacheControl(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
formValues := canonicalFormValues(map[string]string{
|
|
"Cache-Control": "max-age=3600",
|
|
"Expires": "Wed, 21 Oct 2026 07:28:00 GMT",
|
|
"Content-Disposition": `attachment; filename="report.pdf"`,
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
assert.Equal(t, "max-age=3600", r.Header.Get("Cache-Control"))
|
|
assert.Equal(t, "Wed, 21 Oct 2026 07:28:00 GMT", r.Header.Get("Expires"))
|
|
assert.Equal(t, `attachment; filename="report.pdf"`, r.Header.Get("Content-Disposition"))
|
|
}
|
|
|
|
// TestApplyPostPolicyFormHeaders_IgnoresContentType ensures Content-Type is
|
|
// left alone by the helper (it is set by the caller from either the form
|
|
// value or the uploaded file part just before invoking the helper).
|
|
func TestApplyPostPolicyFormHeaders_IgnoresContentType(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodPost, "/bucket", nil)
|
|
r.Header.Set("Content-Type", "image/png")
|
|
formValues := canonicalFormValues(map[string]string{
|
|
"Content-Type": "text/plain",
|
|
})
|
|
|
|
applyPostPolicyFormHeaders(r, formValues)
|
|
|
|
// The helper must not overwrite the Content-Type that the handler has
|
|
// already resolved from the form or from the file part.
|
|
assert.Equal(t, "image/png", r.Header.Get("Content-Type"))
|
|
}
|
|
|
|
// TestPostPolicyBucketHandlerKeyExtraction tests that the handler correctly
|
|
// extracts and normalizes the key from a POST request
|
|
func TestPostPolicyBucketHandlerKeyExtraction(t *testing.T) {
|
|
// Create a minimal S3ApiServer for testing
|
|
s3a := &S3ApiServer{
|
|
option: &S3ApiServerOption{
|
|
BucketsPath: "/buckets",
|
|
},
|
|
iam: &IdentityAccessManagement{},
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
bucket string
|
|
key string
|
|
wantPathHas string // substring that must be in the constructed path
|
|
}{
|
|
{
|
|
name: "key without leading slash",
|
|
bucket: "test-bucket",
|
|
key: "simple-file.txt",
|
|
wantPathHas: "/test-bucket/simple-file.txt",
|
|
},
|
|
{
|
|
name: "key with leading slash",
|
|
bucket: "test-bucket",
|
|
key: "/prefixed-file.txt",
|
|
wantPathHas: "/test-bucket/prefixed-file.txt",
|
|
},
|
|
{
|
|
name: "key with duplicate slashes",
|
|
bucket: "test-bucket",
|
|
key: "folder//nested///file.txt",
|
|
wantPathHas: "/test-bucket/folder/nested/file.txt",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Create multipart form body
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
|
|
// Add required fields
|
|
writer.WriteField("key", tt.key)
|
|
writer.WriteField("Policy", "") // Empty policy for this test
|
|
|
|
// Add file
|
|
part, _ := writer.CreateFormFile("file", "test.txt")
|
|
part.Write([]byte("test content"))
|
|
writer.Close()
|
|
|
|
// Create request
|
|
req := httptest.NewRequest(http.MethodPost, "/"+tt.bucket, &buf)
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
// Set up mux vars (simulating router)
|
|
req = mux.SetURLVars(req, map[string]string{"bucket": tt.bucket})
|
|
|
|
// Parse form to extract key
|
|
reader, _ := req.MultipartReader()
|
|
form, _ := reader.ReadForm(5 * 1024 * 1024)
|
|
defer form.RemoveAll()
|
|
|
|
_, _, _, _, formValues, _ := extractPostPolicyFormValues(form)
|
|
|
|
// Apply the same normalization as PostPolicyBucketHandler
|
|
object := s3_constants.NormalizeObjectKey(formValues.Get("Key"))
|
|
|
|
// Construct path
|
|
filePath := s3a.option.BucketsPath + "/" + tt.bucket + "/" + object
|
|
|
|
assert.Contains(t, filePath, tt.wantPathHas,
|
|
"Path should contain properly separated bucket and key")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPostPolicyBucketHandlerRejectsTraversalKey(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
assert.NoError(t, writer.WriteField("key", "uploads/../../victim-bucket/secret"))
|
|
assert.NoError(t, writer.WriteField("Policy", ""))
|
|
part, err := writer.CreateFormFile("file", "payload.txt")
|
|
assert.NoError(t, err)
|
|
_, err = part.Write([]byte("secret"))
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, writer.Close())
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/source-bucket", &buf)
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
req = mux.SetURLVars(req, map[string]string{"bucket": "source-bucket"})
|
|
rec := httptest.NewRecorder()
|
|
|
|
s3a := &S3ApiServer{
|
|
option: &S3ApiServerOption{BucketsPath: "/buckets"},
|
|
iam: &IdentityAccessManagement{},
|
|
}
|
|
s3a.PostPolicyBucketHandler(rec, req)
|
|
|
|
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
|
assert.Contains(t, rec.Body.String(), "InvalidRequest")
|
|
}
|
|
|
|
// TestPostPolicyBucketHandler_PolicyViolationReturns403 drives the handler
|
|
// end-to-end with a signed multipart POST whose policy conditions cannot be
|
|
// satisfied by the form fields. It verifies the handler responds 403
|
|
// AccessDenied with no Location redirect, rather than the old 307 Temporary
|
|
// Redirect that obscured the policy failure.
|
|
func TestPostPolicyBucketHandler_PolicyViolationReturns403(t *testing.T) {
|
|
const (
|
|
accessKey = "AKIATESTTESTTEST"
|
|
secretKey = "secret-key-for-tests"
|
|
region = "us-east-1"
|
|
service = "s3"
|
|
testBucket = "test-bucket"
|
|
)
|
|
|
|
iam := &IdentityAccessManagement{
|
|
hashes: make(map[string]*sync.Pool),
|
|
hashCounters: make(map[string]*int32),
|
|
}
|
|
err := iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
|
Identities: []*iam_pb.Identity{{
|
|
Name: "tester",
|
|
Credentials: []*iam_pb.Credential{{AccessKey: accessKey, SecretKey: secretKey}},
|
|
Actions: []string{"Admin", "Read", "Write"},
|
|
}},
|
|
})
|
|
assert.NoError(t, err, "loadS3ApiConfiguration should succeed")
|
|
|
|
s3a := &S3ApiServer{
|
|
option: &S3ApiServerOption{BucketsPath: "/buckets"},
|
|
iam: iam,
|
|
}
|
|
// Pre-populate the bucket registry so validateTableBucketObjectPath sees
|
|
// a non-table bucket without needing a live filer connection.
|
|
s3a.bucketRegistry = NewBucketRegistry(s3a)
|
|
s3a.bucketRegistry.setMetadataCache(&BucketMetaData{Name: testBucket, IsTableBucket: false})
|
|
|
|
now := time.Now().UTC()
|
|
amzDate := now.Format(iso8601Format)
|
|
yyyymmddStr := now.Format(yyyymmdd)
|
|
credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", accessKey, yyyymmddStr, region, service)
|
|
expiration := now.Add(1 * time.Hour).Format("2006-01-02T15:04:05.000Z")
|
|
|
|
policyJSON := fmt.Sprintf(
|
|
`{"expiration":"%s","conditions":[`+
|
|
`["eq","$bucket","%s"],`+
|
|
`["eq","$key","required.txt"],`+
|
|
`["eq","$x-amz-credential","%s"],`+
|
|
`["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],`+
|
|
`["eq","$x-amz-date","%s"]`+
|
|
`]}`,
|
|
expiration, testBucket, credential, amzDate,
|
|
)
|
|
encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON))
|
|
|
|
signingKey := getSigningKey(secretKey, yyyymmddStr, region, service)
|
|
signature := getSignature(signingKey, encodedPolicy)
|
|
// Sanity-check: the signature must be valid hex of the expected length so
|
|
// that doesPolicySignatureV4Match does not trip on formatting.
|
|
_, decodeErr := hex.DecodeString(signature)
|
|
assert.NoError(t, decodeErr, "computed signature should be hex-encoded")
|
|
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
assert.NoError(t, writer.WriteField("bucket", testBucket))
|
|
// Deliberately mismatch the policy's required key to force a violation.
|
|
assert.NoError(t, writer.WriteField("key", "wrong.txt"))
|
|
assert.NoError(t, writer.WriteField("x-amz-credential", credential))
|
|
assert.NoError(t, writer.WriteField("x-amz-algorithm", "AWS4-HMAC-SHA256"))
|
|
assert.NoError(t, writer.WriteField("x-amz-date", amzDate))
|
|
assert.NoError(t, writer.WriteField("policy", encodedPolicy))
|
|
assert.NoError(t, writer.WriteField("x-amz-signature", signature))
|
|
|
|
filePart, err := writer.CreateFormFile("file", "payload.txt")
|
|
assert.NoError(t, err)
|
|
_, err = filePart.Write([]byte("contents that should never be uploaded"))
|
|
assert.NoError(t, err)
|
|
assert.NoError(t, writer.Close())
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/"+testBucket, &buf)
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
req = mux.SetURLVars(req, map[string]string{"bucket": testBucket})
|
|
rec := httptest.NewRecorder()
|
|
|
|
s3a.PostPolicyBucketHandler(rec, req)
|
|
|
|
assert.Equal(t, http.StatusForbidden, rec.Code,
|
|
"policy violation must return 403, actual body: %s", rec.Body.String())
|
|
assert.NotEqual(t, http.StatusTemporaryRedirect, rec.Code,
|
|
"must not return 307 Temporary Redirect on policy violation")
|
|
assert.Empty(t, rec.Header().Get("Location"),
|
|
"403 response must not set a redirect Location header")
|
|
assert.Contains(t, rec.Body.String(), "AccessDenied",
|
|
"response body should identify AccessDenied; actual body: %s", rec.Body.String())
|
|
assert.Contains(t, rec.Body.String(), "Policy Condition failed",
|
|
"response body should carry the specific policy-failure message; actual body: %s", rec.Body.String())
|
|
// Guard against the signing setup silently failing before the policy
|
|
// branch ever runs, which would make the 403 assertion meaningless.
|
|
assert.NotContains(t, rec.Body.String(), "SignatureDoesNotMatch",
|
|
"signature must match so the handler reaches the policy-check branch")
|
|
assert.NotContains(t, rec.Body.String(), "InvalidAccessKeyId",
|
|
"access key must be known so the handler reaches the policy-check branch")
|
|
}
|