mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-06 20:17:38 +01:00
weed/s3api: prune test-only functions (#8840)
weed/s3api: prune functions that are referenced only from tests and the tests that exercise them.
This commit is contained in:
Lars Lehtonen authored and GitHub committed
1 parent
955a011f39
commit
5c5d377277
6 files changed
-543
No files matched your search
@@ -1039,13 +1039,6 @@ func getMD5HashBase64(data []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(getMD5Sum(data))
|
||||
}
|
||||
|
||||
// getSHA256Sum returns SHA-256 sum of given data.
|
||||
func getSHA256Sum(data []byte) []byte {
|
||||
hash := sha256.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// getMD5Sum returns MD5 sum of given data.
|
||||
func getMD5Sum(data []byte) []byte {
|
||||
hash := md5.New()
|
||||
@@ -1053,11 +1046,6 @@ func getMD5Sum(data []byte) []byte {
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// getMD5Hash returns MD5 hash in hex encoding of given data.
|
||||
func getMD5Hash(data []byte) string {
|
||||
return hex.EncodeToString(getMD5Sum(data))
|
||||
}
|
||||
|
||||
var ignoredHeaders = map[string]bool{
|
||||
"Authorization": true,
|
||||
"Content-Type": true,
|
||||
|
||||
@@ -539,10 +539,6 @@ func testJWTAuthentication(t *testing.T, iam *IdentityAccessManagement, token st
|
||||
return iam.authenticateJWTWithIAM(req)
|
||||
}
|
||||
|
||||
func testJWTAuthorization(t *testing.T, iam *IdentityAccessManagement, identity *Identity, action Action, bucket, object, token string) bool {
|
||||
return testJWTAuthorizationWithRole(t, iam, identity, action, bucket, object, token, "TestRole")
|
||||
}
|
||||
|
||||
func testJWTAuthorizationWithRole(t *testing.T, iam *IdentityAccessManagement, identity *Identity, action Action, bucket, object, token, roleName string) bool {
|
||||
// Create test request
|
||||
req := httptest.NewRequest("GET", "/"+bucket+"/"+object, http.NoBody)
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// ResponseRecorder that also implements http.Flusher
|
||||
type recorderFlusher struct{ *httptest.ResponseRecorder }
|
||||
|
||||
func (r recorderFlusher) Flush() {}
|
||||
|
||||
// TestSSECRangeRequestsSupported verifies that HTTP Range requests are now supported
|
||||
// for SSE-C encrypted objects since the IV is stored in metadata and CTR mode allows seeking
|
||||
func TestSSECRangeRequestsSupported(t *testing.T) {
|
||||
// Create a request with Range header and valid SSE-C headers
|
||||
req := httptest.NewRequest(http.MethodGet, "/b/o", nil)
|
||||
req.Header.Set("Range", "bytes=10-20")
|
||||
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
|
||||
|
||||
key := make([]byte, 32)
|
||||
for i := range key {
|
||||
key[i] = byte(i)
|
||||
}
|
||||
s := md5.Sum(key)
|
||||
keyMD5 := base64.StdEncoding.EncodeToString(s[:])
|
||||
|
||||
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key))
|
||||
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyMD5)
|
||||
|
||||
// Attach mux vars to avoid panic in error writer
|
||||
req = mux.SetURLVars(req, map[string]string{"bucket": "b", "object": "o"})
|
||||
|
||||
// Create a mock HTTP response that simulates SSE-C encrypted object metadata
|
||||
proxyResponse := &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(bytes.NewReader([]byte("mock encrypted data"))),
|
||||
}
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyMD5)
|
||||
|
||||
// Call the function under test - should no longer reject range requests
|
||||
s3a := &S3ApiServer{
|
||||
option: &S3ApiServerOption{
|
||||
BucketsPath: "/buckets",
|
||||
},
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
w := recorderFlusher{rec}
|
||||
// Pass nil for entry since this test focuses on Range request handling
|
||||
statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w, nil)
|
||||
|
||||
// Range requests should now be allowed to proceed (will be handled by filer layer)
|
||||
// The exact status code depends on the object existence and filer response
|
||||
if statusCode == http.StatusRequestedRangeNotSatisfiable {
|
||||
t.Fatalf("Range requests should no longer be rejected for SSE-C objects, got status %d", statusCode)
|
||||
}
|
||||
}
|
||||
@@ -331,47 +331,6 @@ func TestDetectPrimarySSETypeS3(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAddSSES3HeadersToResponse tests that SSE-S3 headers are added to responses
|
||||
func TestAddSSES3HeadersToResponse(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.AmzServerSideEncryption: []byte("AES256"),
|
||||
},
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{
|
||||
FileId: "1,123",
|
||||
Offset: 0,
|
||||
Size: 1024,
|
||||
SseType: filer_pb.SSEType_SSE_S3,
|
||||
SseMetadata: []byte("metadata"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
proxyResponse := &http.Response{
|
||||
Header: make(http.Header),
|
||||
}
|
||||
|
||||
s3a.addSSEHeadersToResponse(proxyResponse, entry)
|
||||
|
||||
algorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryption)
|
||||
if algorithm != "AES256" {
|
||||
t.Errorf("Expected SSE algorithm AES256, got %s", algorithm)
|
||||
}
|
||||
|
||||
// Should NOT have SSE-C or SSE-KMS specific headers
|
||||
if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm) != "" {
|
||||
t.Error("Should not have SSE-C customer algorithm header")
|
||||
}
|
||||
|
||||
if proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId) != "" {
|
||||
t.Error("Should not have SSE-KMS key ID header")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSES3EncryptionWithBaseIV tests multipart encryption with base IV
|
||||
func TestSSES3EncryptionWithBaseIV(t *testing.T) {
|
||||
// Generate SSE-S3 key
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/mem"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
@@ -2394,45 +2393,6 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func captureCORSHeaders(w http.ResponseWriter, headersToCapture []string) map[string]string {
|
||||
captured := make(map[string]string)
|
||||
for _, corsHeader := range headersToCapture {
|
||||
if value := w.Header().Get(corsHeader); value != "" {
|
||||
captured[corsHeader] = value
|
||||
}
|
||||
}
|
||||
return captured
|
||||
}
|
||||
|
||||
func restoreCORSHeaders(w http.ResponseWriter, capturedCORSHeaders map[string]string) {
|
||||
for corsHeader, value := range capturedCORSHeaders {
|
||||
w.Header().Set(corsHeader, value)
|
||||
}
|
||||
}
|
||||
|
||||
// writeFinalResponse handles the common response writing logic shared between
|
||||
// passThroughResponse and handleSSECResponse
|
||||
func writeFinalResponse(w http.ResponseWriter, proxyResponse *http.Response, bodyReader io.Reader, capturedCORSHeaders map[string]string) (statusCode int, bytesTransferred int64) {
|
||||
// Restore CORS headers that were set by middleware
|
||||
restoreCORSHeaders(w, capturedCORSHeaders)
|
||||
|
||||
if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
|
||||
statusCode = http.StatusPartialContent
|
||||
} else {
|
||||
statusCode = proxyResponse.StatusCode
|
||||
}
|
||||
w.WriteHeader(statusCode)
|
||||
|
||||
// Stream response data
|
||||
buf := mem.Allocate(128 * 1024)
|
||||
defer mem.Free(buf)
|
||||
bytesTransferred, err := io.CopyBuffer(w, bodyReader, buf)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("response read %d bytes: %v", bytesTransferred, err)
|
||||
}
|
||||
return statusCode, bytesTransferred
|
||||
}
|
||||
|
||||
// fetchObjectEntry fetches the filer entry for an object
|
||||
// Returns nil if not found (not an error), or propagates other errors
|
||||
func (s3a *S3ApiServer) fetchObjectEntry(bucket, object string) (*filer_pb.Entry, error) {
|
||||
@@ -2458,187 +2418,6 @@ func (s3a *S3ApiServer) fetchObjectEntryRequired(bucket, object string) (*filer_
|
||||
return fetchedEntry, nil
|
||||
}
|
||||
|
||||
// copyResponseHeaders copies headers from proxy response to the response writer,
|
||||
// excluding internal SeaweedFS headers and optionally excluding body-related headers
|
||||
func copyResponseHeaders(w http.ResponseWriter, proxyResponse *http.Response, excludeBodyHeaders bool) {
|
||||
for k, v := range proxyResponse.Header {
|
||||
// Always exclude internal SeaweedFS headers
|
||||
if s3_constants.IsSeaweedFSInternalHeader(k) {
|
||||
continue
|
||||
}
|
||||
// Optionally exclude body-related headers that might change after decryption
|
||||
if excludeBodyHeaders && (k == "Content-Length" || k == "Content-Encoding") {
|
||||
continue
|
||||
}
|
||||
w.Header()[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int, bytesTransferred int64) {
|
||||
// Capture existing CORS headers that may have been set by middleware
|
||||
capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
|
||||
|
||||
// Copy headers from proxy response (excluding internal SeaweedFS headers)
|
||||
copyResponseHeaders(w, proxyResponse, false)
|
||||
|
||||
return writeFinalResponse(w, proxyResponse, proxyResponse.Body, capturedCORSHeaders)
|
||||
}
|
||||
|
||||
// handleSSECResponse handles SSE-C decryption and response processing
|
||||
func (s3a *S3ApiServer) handleSSECResponse(r *http.Request, proxyResponse *http.Response, w http.ResponseWriter, entry *filer_pb.Entry) (statusCode int, bytesTransferred int64) {
|
||||
// Check if the object has SSE-C metadata
|
||||
sseAlgorithm := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
sseKeyMD5 := proxyResponse.Header.Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
isObjectEncrypted := sseAlgorithm != "" && sseKeyMD5 != ""
|
||||
|
||||
// Parse SSE-C headers from request once (avoid duplication)
|
||||
customerKey, err := ParseSSECHeaders(r)
|
||||
if err != nil {
|
||||
errCode := MapSSECErrorToS3Error(err)
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return http.StatusBadRequest, 0
|
||||
}
|
||||
|
||||
if isObjectEncrypted {
|
||||
// This object was encrypted with SSE-C, validate customer key
|
||||
if customerKey == nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyMissing)
|
||||
return http.StatusBadRequest, 0
|
||||
}
|
||||
|
||||
// SSE-C MD5 is base64 and case-sensitive
|
||||
if customerKey.KeyMD5 != sseKeyMD5 {
|
||||
// For GET/HEAD requests, AWS S3 returns 403 Forbidden for a key mismatch.
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return http.StatusForbidden, 0
|
||||
}
|
||||
|
||||
// SSE-C encrypted objects support HTTP Range requests
|
||||
// The IV is stored in metadata and CTR mode allows seeking to any offset
|
||||
// Range requests will be handled by the filer layer with proper offset-based decryption
|
||||
|
||||
// Check if this is a chunked or small content SSE-C object
|
||||
// Use the entry parameter passed from the caller (avoids redundant lookup)
|
||||
if entry != nil {
|
||||
// Check for SSE-C chunks
|
||||
sseCChunks := 0
|
||||
for _, chunk := range entry.GetChunks() {
|
||||
if chunk.GetSseType() == filer_pb.SSEType_SSE_C {
|
||||
sseCChunks++
|
||||
}
|
||||
}
|
||||
|
||||
if sseCChunks >= 1 {
|
||||
|
||||
// Handle chunked SSE-C objects - each chunk needs independent decryption
|
||||
multipartReader, decErr := s3a.createMultipartSSECDecryptedReader(r, proxyResponse, entry)
|
||||
if decErr != nil {
|
||||
glog.Errorf("Failed to create multipart SSE-C decrypted reader: %v", decErr)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return http.StatusInternalServerError, 0
|
||||
}
|
||||
|
||||
// Capture existing CORS headers
|
||||
capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
|
||||
|
||||
// Copy headers from proxy response (excluding internal SeaweedFS headers)
|
||||
copyResponseHeaders(w, proxyResponse, false)
|
||||
|
||||
// Set proper headers for range requests
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
|
||||
// Parse range header (e.g., "bytes=0-99")
|
||||
if len(rangeHeader) > 6 && rangeHeader[:6] == "bytes=" {
|
||||
rangeSpec := rangeHeader[6:]
|
||||
parts := strings.Split(rangeSpec, "-")
|
||||
if len(parts) == 2 {
|
||||
startOffset, endOffset := int64(0), int64(-1)
|
||||
if parts[0] != "" {
|
||||
startOffset, _ = strconv.ParseInt(parts[0], 10, 64)
|
||||
}
|
||||
if parts[1] != "" {
|
||||
endOffset, _ = strconv.ParseInt(parts[1], 10, 64)
|
||||
}
|
||||
|
||||
if endOffset >= startOffset {
|
||||
// Specific range - set proper Content-Length and Content-Range headers
|
||||
rangeLength := endOffset - startOffset + 1
|
||||
totalSize := proxyResponse.Header.Get("Content-Length")
|
||||
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(rangeLength, 10))
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%s", startOffset, endOffset, totalSize))
|
||||
// writeFinalResponse will set status to 206 if Content-Range is present
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return writeFinalResponse(w, proxyResponse, multipartReader, capturedCORSHeaders)
|
||||
} else if len(entry.GetChunks()) == 0 && len(entry.Content) > 0 {
|
||||
// Small content SSE-C object stored directly in entry.Content
|
||||
|
||||
// Fall through to traditional single-object SSE-C handling below
|
||||
}
|
||||
}
|
||||
|
||||
// Single-part SSE-C object: Get IV from proxy response headers (stored during upload)
|
||||
ivBase64 := proxyResponse.Header.Get(s3_constants.SeaweedFSSSEIVHeader)
|
||||
if ivBase64 == "" {
|
||||
glog.Errorf("SSE-C encrypted single-part object missing IV in metadata")
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return http.StatusInternalServerError, 0
|
||||
}
|
||||
|
||||
iv, err := base64.StdEncoding.DecodeString(ivBase64)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to decode IV from metadata: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return http.StatusInternalServerError, 0
|
||||
}
|
||||
|
||||
// Create decrypted reader with IV from metadata
|
||||
decryptedReader, decErr := CreateSSECDecryptedReader(proxyResponse.Body, customerKey, iv)
|
||||
if decErr != nil {
|
||||
glog.Errorf("Failed to create SSE-C decrypted reader: %v", decErr)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return http.StatusInternalServerError, 0
|
||||
}
|
||||
|
||||
// Capture existing CORS headers that may have been set by middleware
|
||||
capturedCORSHeaders := captureCORSHeaders(w, corsHeaders)
|
||||
|
||||
// Copy headers from proxy response (excluding body-related headers that might change and internal SeaweedFS headers)
|
||||
copyResponseHeaders(w, proxyResponse, true)
|
||||
|
||||
// Set correct Content-Length for SSE-C (only for full object requests)
|
||||
// With IV stored in metadata, the encrypted length equals the original length
|
||||
if proxyResponse.Header.Get("Content-Range") == "" {
|
||||
// Full object request: encrypted length equals original length (IV not in stream)
|
||||
if contentLengthStr := proxyResponse.Header.Get("Content-Length"); contentLengthStr != "" {
|
||||
// Content-Length is already correct since IV is stored in metadata, not in data stream
|
||||
w.Header().Set("Content-Length", contentLengthStr)
|
||||
}
|
||||
}
|
||||
// For range requests, let the actual bytes transferred determine the response length
|
||||
|
||||
// Add SSE-C response headers
|
||||
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, sseAlgorithm)
|
||||
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, sseKeyMD5)
|
||||
|
||||
return writeFinalResponse(w, proxyResponse, decryptedReader, capturedCORSHeaders)
|
||||
} else {
|
||||
// Object is not encrypted, but check if customer provided SSE-C headers unnecessarily
|
||||
if customerKey != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrSSECustomerKeyNotNeeded)
|
||||
return http.StatusBadRequest, 0
|
||||
}
|
||||
|
||||
// Normal pass-through response
|
||||
return passThroughResponse(proxyResponse, w)
|
||||
}
|
||||
}
|
||||
|
||||
// addObjectLockHeadersToResponse extracts object lock metadata from entry Extended attributes
|
||||
// and adds the appropriate S3 headers to the response
|
||||
func (s3a *S3ApiServer) addObjectLockHeadersToResponse(w http.ResponseWriter, entry *filer_pb.Entry) {
|
||||
@@ -2680,54 +2459,6 @@ func (s3a *S3ApiServer) addObjectLockHeadersToResponse(w http.ResponseWriter, en
|
||||
}
|
||||
}
|
||||
|
||||
// addSSEHeadersToResponse converts stored SSE metadata from entry.Extended to HTTP response headers
|
||||
// Uses intelligent prioritization: only set headers for the PRIMARY encryption type to avoid conflicts
|
||||
func (s3a *S3ApiServer) addSSEHeadersToResponse(proxyResponse *http.Response, entry *filer_pb.Entry) {
|
||||
if entry == nil || entry.Extended == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Determine the primary encryption type by examining chunks (most reliable)
|
||||
primarySSEType := s3a.detectPrimarySSEType(entry)
|
||||
|
||||
// Only set headers for the PRIMARY encryption type
|
||||
switch primarySSEType {
|
||||
case s3_constants.SSETypeC:
|
||||
// Add only SSE-C headers
|
||||
if algorithmBytes, exists := entry.Extended[s3_constants.AmzServerSideEncryptionCustomerAlgorithm]; exists && len(algorithmBytes) > 0 {
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, string(algorithmBytes))
|
||||
}
|
||||
|
||||
if keyMD5Bytes, exists := entry.Extended[s3_constants.AmzServerSideEncryptionCustomerKeyMD5]; exists && len(keyMD5Bytes) > 0 {
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, string(keyMD5Bytes))
|
||||
}
|
||||
|
||||
if ivBytes, exists := entry.Extended[s3_constants.SeaweedFSSSEIV]; exists && len(ivBytes) > 0 {
|
||||
ivBase64 := base64.StdEncoding.EncodeToString(ivBytes)
|
||||
proxyResponse.Header.Set(s3_constants.SeaweedFSSSEIVHeader, ivBase64)
|
||||
}
|
||||
|
||||
case s3_constants.SSETypeKMS:
|
||||
// Add only SSE-KMS headers
|
||||
if sseAlgorithm, exists := entry.Extended[s3_constants.AmzServerSideEncryption]; exists && len(sseAlgorithm) > 0 {
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryption, string(sseAlgorithm))
|
||||
}
|
||||
|
||||
if kmsKeyID, exists := entry.Extended[s3_constants.AmzServerSideEncryptionAwsKmsKeyId]; exists && len(kmsKeyID) > 0 {
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, string(kmsKeyID))
|
||||
}
|
||||
|
||||
case s3_constants.SSETypeS3:
|
||||
// Add only SSE-S3 headers
|
||||
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryption, SSES3Algorithm)
|
||||
|
||||
default:
|
||||
// Unencrypted or unknown - don't set any SSE headers
|
||||
}
|
||||
|
||||
glog.V(3).Infof("addSSEHeadersToResponse: processed %d extended metadata entries", len(entry.Extended))
|
||||
}
|
||||
|
||||
// detectPrimarySSEType determines the primary SSE type by examining chunk metadata
|
||||
func (s3a *S3ApiServer) detectPrimarySSEType(entry *filer_pb.Entry) string {
|
||||
// Safety check: handle nil entry
|
||||
@@ -3183,140 +2914,6 @@ func (r *SSERangeReader) Read(p []byte) (n int, err error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// createMultipartSSECDecryptedReader creates a decrypted reader for multipart SSE-C objects
|
||||
// Each chunk has its own IV and encryption key from the original multipart parts
|
||||
func (s3a *S3ApiServer) createMultipartSSECDecryptedReader(r *http.Request, proxyResponse *http.Response, entry *filer_pb.Entry) (io.Reader, error) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Parse SSE-C headers from the request for decryption key
|
||||
customerKey, err := ParseSSECHeaders(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid SSE-C headers for multipart decryption: %v", err)
|
||||
}
|
||||
|
||||
// Entry is passed from caller to avoid redundant filer lookup
|
||||
|
||||
// Sort chunks by offset to ensure correct order
|
||||
chunks := entry.GetChunks()
|
||||
sort.Slice(chunks, func(i, j int) bool {
|
||||
return chunks[i].GetOffset() < chunks[j].GetOffset()
|
||||
})
|
||||
|
||||
// Check for Range header to optimize chunk processing
|
||||
var startOffset, endOffset int64 = 0, -1
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
// Parse range header (e.g., "bytes=0-99")
|
||||
if len(rangeHeader) > 6 && rangeHeader[:6] == "bytes=" {
|
||||
rangeSpec := rangeHeader[6:]
|
||||
parts := strings.Split(rangeSpec, "-")
|
||||
if len(parts) == 2 {
|
||||
if parts[0] != "" {
|
||||
startOffset, _ = strconv.ParseInt(parts[0], 10, 64)
|
||||
}
|
||||
if parts[1] != "" {
|
||||
endOffset, _ = strconv.ParseInt(parts[1], 10, 64)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter chunks to only those needed for the range request
|
||||
var neededChunks []*filer_pb.FileChunk
|
||||
for _, chunk := range chunks {
|
||||
chunkStart := chunk.GetOffset()
|
||||
chunkEnd := chunkStart + int64(chunk.GetSize()) - 1
|
||||
|
||||
// Check if this chunk overlaps with the requested range
|
||||
if endOffset == -1 {
|
||||
// No end specified, take all chunks from startOffset
|
||||
if chunkEnd >= startOffset {
|
||||
neededChunks = append(neededChunks, chunk)
|
||||
}
|
||||
} else {
|
||||
// Specific range: check for overlap
|
||||
if chunkStart <= endOffset && chunkEnd >= startOffset {
|
||||
neededChunks = append(neededChunks, chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create readers for only the needed chunks
|
||||
var readers []io.Reader
|
||||
|
||||
for _, chunk := range neededChunks {
|
||||
|
||||
// Get this chunk's encrypted data
|
||||
chunkReader, err := s3a.createEncryptedChunkReader(ctx, chunk)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create chunk reader: %v", err)
|
||||
}
|
||||
|
||||
if chunk.GetSseType() == filer_pb.SSEType_SSE_C {
|
||||
// For SSE-C chunks, extract the IV from the stored per-chunk metadata (unified approach)
|
||||
if len(chunk.GetSseMetadata()) > 0 {
|
||||
// Deserialize the SSE-C metadata stored in the unified metadata field
|
||||
ssecMetadata, decErr := DeserializeSSECMetadata(chunk.GetSseMetadata())
|
||||
if decErr != nil {
|
||||
chunkReader.Close()
|
||||
return nil, fmt.Errorf("failed to deserialize SSE-C metadata for chunk %s: %v", chunk.GetFileIdString(), decErr)
|
||||
}
|
||||
|
||||
// Decode the IV from the metadata
|
||||
iv, ivErr := base64.StdEncoding.DecodeString(ssecMetadata.IV)
|
||||
if ivErr != nil {
|
||||
chunkReader.Close()
|
||||
return nil, fmt.Errorf("failed to decode IV for SSE-C chunk %s: %v", chunk.GetFileIdString(), ivErr)
|
||||
}
|
||||
|
||||
partOffset := ssecMetadata.PartOffset
|
||||
if partOffset < 0 {
|
||||
chunkReader.Close()
|
||||
return nil, fmt.Errorf("invalid SSE-C part offset %d for chunk %s", partOffset, chunk.GetFileIdString())
|
||||
}
|
||||
|
||||
// Use stored IV and advance CTR stream by PartOffset within the encrypted stream
|
||||
decryptedReader, decErr := CreateSSECDecryptedReaderWithOffset(chunkReader, customerKey, iv, uint64(partOffset))
|
||||
if decErr != nil {
|
||||
chunkReader.Close()
|
||||
return nil, fmt.Errorf("failed to create SSE-C decrypted reader for chunk %s: %v", chunk.GetFileIdString(), decErr)
|
||||
}
|
||||
readers = append(readers, decryptedReader)
|
||||
} else {
|
||||
chunkReader.Close()
|
||||
return nil, fmt.Errorf("SSE-C chunk %s missing required metadata", chunk.GetFileIdString())
|
||||
}
|
||||
} else {
|
||||
// Non-SSE-C chunk, use as-is
|
||||
readers = append(readers, chunkReader)
|
||||
}
|
||||
}
|
||||
|
||||
multiReader := NewMultipartSSEReader(readers)
|
||||
|
||||
// Apply range logic if a range was requested
|
||||
if rangeHeader != "" && startOffset >= 0 {
|
||||
if endOffset == -1 {
|
||||
// Open-ended range (e.g., "bytes=100-")
|
||||
return &SSERangeReader{
|
||||
reader: multiReader,
|
||||
offset: startOffset,
|
||||
remaining: -1, // Read until EOF
|
||||
}, nil
|
||||
} else {
|
||||
// Specific range (e.g., "bytes=0-99")
|
||||
rangeLength := endOffset - startOffset + 1
|
||||
return &SSERangeReader{
|
||||
reader: multiReader,
|
||||
offset: startOffset,
|
||||
remaining: rangeLength,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return multiReader, nil
|
||||
}
|
||||
|
||||
// PartBoundaryInfo holds information about a part's chunk boundaries
|
||||
type PartBoundaryInfo struct {
|
||||
PartNumber int `json:"part"`
|
||||
|
||||
@@ -9,22 +9,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// mockAccountManager implements AccountManager for testing
|
||||
type mockAccountManager struct {
|
||||
accounts map[string]string
|
||||
}
|
||||
|
||||
func (m *mockAccountManager) GetAccountNameById(id string) string {
|
||||
if name, exists := m.accounts[id]; exists {
|
||||
return name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *mockAccountManager) GetAccountIdByEmail(email string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestNewListEntryOwnerDisplayName(t *testing.T) {
|
||||
// Create S3ApiServer with a properly initialized IAM
|
||||
s3a := &S3ApiServer{
|
||||
|
||||
Reference in new issue
Block a user