mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
operation: prepare upload content before assigning the volume (#12714)
* operation: prepare upload content before assigning the volume * operation: keep the existing upload function names for prepared payloads UploadData and doUpload prepare once and hand the prepared payload to retriedUploadData, so the retry loop, doUploadData, uploadChunk and uploadChunkToHolders take a preparedUpload directly instead of growing a prepared twin of each and branching on nil. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: send ciphertext whenever the prepared payload carries a cipher key doUploadData chose the cipher branch from the option, so a payload prepared under a different option could be sent as plaintext metadata or lose its key. Decide from the prepared payload itself. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: build a chunk's preparation and upload options in one place The chunk loop prepared with a hand-written option while uploadChunk and uploadChunkToHolders built their own, so a setting that shapes the bytes could reach one and not the other. chunkUploadOption now feeds all three. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: test that a reassigned chunk reuses its prepared payload Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
4 files changed
+489
-64
No files matched your search
@@ -0,0 +1,370 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type preparationAssignClient struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
assign func(*filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse
|
||||
}
|
||||
|
||||
func (c *preparationAssignClient) AssignVolume(_ context.Context, req *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) {
|
||||
return c.assign(req), nil
|
||||
}
|
||||
func (c *preparationAssignClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return fn(c)
|
||||
}
|
||||
func (c *preparationAssignClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url }
|
||||
func (c *preparationAssignClient) GetDataCenter() string { return "" }
|
||||
|
||||
// The preparation collaborator performs real gzip/encryption and advances only
|
||||
// the JWT verifier's clock. No wall-clock sleep or slow-machine threshold is used.
|
||||
func TestUploadPreparesBeforeAssignment(t *testing.T) {
|
||||
for _, cipher := range []bool{false, true} {
|
||||
name := "gzip"
|
||||
if cipher {
|
||||
name = "gzip_cipher"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte{0}, 128*1024)
|
||||
sum := md5.Sum(payload)
|
||||
digest := base64.StdEncoding.EncodeToString(sum[:])
|
||||
const fid = "3,01637037d6"
|
||||
key := []byte("disposable uploader boundary test")
|
||||
var now atomic.Int64
|
||||
now.Store(1700000000)
|
||||
var requests atomic.Int32
|
||||
var stored []*needle.Needle
|
||||
var mu sync.Mutex
|
||||
var parseFailure atomic.Bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
claims := &security.SeaweedFileIdClaims{}
|
||||
token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"}))
|
||||
if err != nil || !token.Valid || claims.Fid != fid {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" {
|
||||
parseFailure.Store(true)
|
||||
}
|
||||
n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
parseFailure.Store(true)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if cipher {
|
||||
if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 {
|
||||
parseFailure.Store(true)
|
||||
}
|
||||
} else if gotMd5 != digest {
|
||||
parseFailure.Store(true)
|
||||
}
|
||||
mu.Lock()
|
||||
stored = append(stored, n)
|
||||
first := len(stored) == 1
|
||||
mu.Unlock()
|
||||
if first {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = io.WriteString(w, `{"error":"busy"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-MD5", digest)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{"name":"long.bin","size":131072}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
uploader := newUploader(server.Client())
|
||||
preparations := 0
|
||||
uploader.prepareData = func(data []byte, opt *UploadOption) (*preparedUpload, error) {
|
||||
preparations++
|
||||
prepared, err := prepareUploadData(data, opt)
|
||||
now.Add(11)
|
||||
return prepared, err
|
||||
}
|
||||
assignments := 0
|
||||
client := &preparationAssignClient{assign: func(req *filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse {
|
||||
assignments++
|
||||
if req.ExpectedDataSize != uint64(len(payload)) {
|
||||
t.Error("assignment did not receive clear size")
|
||||
}
|
||||
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key)
|
||||
if err != nil {
|
||||
t.Fatal("could not sign disposable fixture")
|
||||
}
|
||||
return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: encoded, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}, Fsync: true}
|
||||
}}
|
||||
option := &UploadOption{Filename: "long.bin", Cipher: cipher, MimeType: "application/octet-stream", WantMd5: true, PairMap: map[string]string{"test": "private-metadata"}}
|
||||
gotFid, result, err, clear := uploader.UploadWithRetry(client, &filer_pb.AssignVolumeRequest{Count: 1}, option, bytes.NewReader(payload))
|
||||
mu.Lock()
|
||||
saved := append([]*needle.Needle(nil), stored...)
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("prepared upload was rejected: assignments=%d requests=%d preparations=%d", assignments, requests.Load(), preparations)
|
||||
}
|
||||
if gotFid != fid || !bytes.Equal(clear, payload) || result.Size != uint32(len(payload)) || result.Gzip != 1 || result.ContentMd5 != digest {
|
||||
t.Fatal("clear result metadata changed")
|
||||
}
|
||||
if assignments != 1 || preparations != 1 || requests.Load() != 2 || len(saved) != 2 || parseFailure.Load() {
|
||||
t.Fatalf("outbound/retry contract mismatch: assignments=%d preparations=%d requests=%d", assignments, preparations, requests.Load())
|
||||
}
|
||||
if !bytes.Equal(saved[0].Data, saved[1].Data) {
|
||||
t.Fatal("retry changed prepared bytes")
|
||||
}
|
||||
wire := saved[1].Data
|
||||
if cipher {
|
||||
if len(result.CipherKey) == 0 {
|
||||
t.Fatal("missing cipher key")
|
||||
}
|
||||
wire, err = util.Decrypt(wire, util.CipherKey(result.CipherKey))
|
||||
if err != nil {
|
||||
t.Fatal("cipher payload cannot be decrypted")
|
||||
}
|
||||
}
|
||||
decoded, err := util.DecompressData(wire)
|
||||
if err != nil || !bytes.Equal(decoded, payload) {
|
||||
t.Fatal("prepared wire payload changed clear content")
|
||||
}
|
||||
if option.MimeType != "application/octet-stream" {
|
||||
t.Fatal("preparation mutated caller MIME option")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunksPrepareBeforeAssignment(t *testing.T) {
|
||||
for _, mode := range []string{"relay", "fanout", "cipher"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
cipher := mode == "cipher"
|
||||
payload := bytes.Repeat([]byte{0}, 128*1024)
|
||||
sum := md5.Sum(payload)
|
||||
digest := base64.StdEncoding.EncodeToString(sum[:])
|
||||
const fid = "3,01637037d6"
|
||||
key := []byte("disposable chunk boundary test")
|
||||
var now atomic.Int64
|
||||
now.Store(1700000000)
|
||||
var posts, deletes, preparations, assignments atomic.Int32
|
||||
var invalid atomic.Bool
|
||||
var mu sync.Mutex
|
||||
var stored []*needle.Needle
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodDelete {
|
||||
deletes.Add(1)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
posts.Add(1)
|
||||
claims := &security.SeaweedFileIdClaims{}
|
||||
token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"}))
|
||||
if err != nil || !token.Valid || claims.Fid != fid {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" {
|
||||
invalid.Store(true)
|
||||
}
|
||||
if mode == "fanout" && r.URL.Query().Get("type") != "replicate" {
|
||||
invalid.Store(true)
|
||||
}
|
||||
n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
invalid.Store(true)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if cipher {
|
||||
if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 {
|
||||
invalid.Store(true)
|
||||
}
|
||||
} else if gotMd5 != digest {
|
||||
invalid.Store(true)
|
||||
}
|
||||
mu.Lock()
|
||||
stored = append(stored, n)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-MD5", digest)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{"size":131072}`)
|
||||
})
|
||||
primary := httptest.NewServer(handler)
|
||||
defer primary.Close()
|
||||
replica := httptest.NewServer(handler)
|
||||
defer replica.Close()
|
||||
option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(ctx context.Context, count int, size uint64) (*VolumeAssignRequest, *AssignResult, error) {
|
||||
assignments.Add(1)
|
||||
if count != 1 || size != uint64(len(payload)) {
|
||||
invalid.Store(true)
|
||||
}
|
||||
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
result := &AssignResult{Fid: fid, Url: strings.TrimPrefix(primary.URL, "http://"), Auth: security.EncodedJwt(encoded), Fsync: true}
|
||||
if mode == "fanout" {
|
||||
result.Replicas = []Location{{Url: strings.TrimPrefix(replica.URL, "http://")}}
|
||||
}
|
||||
return nil, result, nil
|
||||
}}
|
||||
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) {
|
||||
preparations.Add(1)
|
||||
prepared, err := prepareUploadData(data, opt)
|
||||
now.Add(11)
|
||||
return prepared, err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("chunk JWT expired during preparation: assignments=%d posts=%d preparations=%d", assignments.Load(), posts.Load(), preparations.Load())
|
||||
}
|
||||
wantPosts := int32(1)
|
||||
if mode == "fanout" {
|
||||
wantPosts = 2
|
||||
}
|
||||
if invalid.Load() || assignments.Load() != 1 || preparations.Load() != 1 || posts.Load() != wantPosts || deletes.Load() != 0 || len(stored) != int(wantPosts) {
|
||||
t.Fatal("chunk assignment/holder contract changed")
|
||||
}
|
||||
if len(result.FileChunks) != 1 || result.TotalSize != int64(len(payload)) || result.FileChunks[0].Size != uint64(len(payload)) || result.FileChunks[0].ETag != digest || !result.FileChunks[0].IsCompressed || result.FileChunks[0].FileId != fid {
|
||||
t.Fatal("chunk clear metadata changed")
|
||||
}
|
||||
if len(stored) == 2 && !bytes.Equal(stored[0].Data, stored[1].Data) {
|
||||
t.Fatal("holders received different prepared payloads")
|
||||
}
|
||||
wire := stored[0].Data
|
||||
if cipher {
|
||||
wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey))
|
||||
if err != nil {
|
||||
t.Fatal("cannot decrypt chunk")
|
||||
}
|
||||
}
|
||||
decoded, err := util.DecompressData(wire)
|
||||
if err != nil || !bytes.Equal(decoded, payload) {
|
||||
t.Fatal("stored chunk differs from clear input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunksPreserveCustomUploadContract(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte{0}, 32768)
|
||||
calls := 0
|
||||
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), &ChunkedUploadOption{ChunkSize: 65536, Cipher: true, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) {
|
||||
return nil, &AssignResult{Fid: "3,01637037d6", Url: "unused", Auth: "disposable-token", Fsync: true}, nil
|
||||
}, UploadFunc: func(_ context.Context, data []byte, opt *UploadOption) (*UploadResult, error) {
|
||||
calls++
|
||||
if !bytes.Equal(data, payload) || !opt.Cipher || opt.IsInputCompressed || opt.MimeType != "application/octet-stream" || opt.Jwt != "disposable-token" || opt.UploadUrl != "http://unused/3,01637037d6?fsync=true" || opt.Md5 == "" || opt.MaxAttempts != 1 {
|
||||
t.Error("custom uploader contract changed")
|
||||
}
|
||||
return &UploadResult{Size: uint32(len(data))}, nil
|
||||
}}, func([]byte, *UploadOption) (*preparedUpload, error) {
|
||||
t.Fatal("custom uploader must receive original clear bytes without internal preparation")
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil || calls != 1 || len(result.FileChunks) != 1 {
|
||||
t.Fatal("custom upload did not complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadChunksReassignReusesPreparedPayload(t *testing.T) {
|
||||
for _, cipher := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "relay", true: "cipher"}[cipher], func(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte{0}, 128*1024)
|
||||
sum := md5.Sum(payload)
|
||||
digest := base64.StdEncoding.EncodeToString(sum[:])
|
||||
fids := []string{"3,01637037d6", "4,02637037d6"}
|
||||
key := []byte("disposable reassign test")
|
||||
var mu sync.Mutex
|
||||
var bodies [][]byte
|
||||
var postedFids, deletedFids []string
|
||||
var preparations, assignments atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fid := strings.TrimPrefix(r.URL.Path, "/")
|
||||
if r.Method == http.MethodDelete {
|
||||
mu.Lock()
|
||||
deletedFids = append(deletedFids, fid)
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
claims := &security.SeaweedFileIdClaims{}
|
||||
if _, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }); err != nil || claims.Fid != fid {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
n, _, _, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
bodies = append(bodies, n.Data)
|
||||
postedFids = append(postedFids, fid)
|
||||
mu.Unlock()
|
||||
if fid == fids[0] {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = io.WriteString(w, `{"error":"volume unavailable"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-MD5", digest)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = io.WriteString(w, `{"size":131072}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) {
|
||||
fid := fids[assignments.Add(1)-1]
|
||||
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid}).SignedString(key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nil, &AssignResult{Fid: fid, Url: strings.TrimPrefix(server.URL, "http://"), Auth: security.EncodedJwt(encoded)}, nil
|
||||
}}
|
||||
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) {
|
||||
preparations.Add(1)
|
||||
return prepareUploadData(data, opt)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upload after reassignment: %v", err)
|
||||
}
|
||||
if preparations.Load() != 1 || assignments.Load() != 2 {
|
||||
t.Fatalf("preparations=%d assignments=%d, want 1 and 2", preparations.Load(), assignments.Load())
|
||||
}
|
||||
if len(bodies) != 2 || !bytes.Equal(bodies[0], bodies[1]) || postedFids[0] != fids[0] || postedFids[1] != fids[1] {
|
||||
t.Fatalf("posted fids %v; want the same prepared bytes on both assignments", postedFids)
|
||||
}
|
||||
if len(deletedFids) != 1 || deletedFids[0] != fids[0] {
|
||||
t.Fatalf("deleted fids %v, want only the failed %s", deletedFids, fids[0])
|
||||
}
|
||||
if len(result.FileChunks) != 1 || result.FileChunks[0].FileId != fids[1] {
|
||||
t.Fatal("chunk does not name the reassigned fid")
|
||||
}
|
||||
wire := bodies[1]
|
||||
if cipher {
|
||||
if wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey)); err != nil {
|
||||
t.Fatalf("decrypt reassigned chunk: %v", err)
|
||||
}
|
||||
}
|
||||
if decoded, err := util.DecompressData(wire); err != nil || !bytes.Equal(decoded, payload) {
|
||||
t.Fatal("reassigned chunk differs from clear input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,10 @@ var chunkBufferPool = sync.Pool{
|
||||
// This prevents OOM by processing the stream in fixed-size chunks
|
||||
// Returns file chunks, MD5 hash, total size, and any small content stored inline
|
||||
func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption) (*ChunkedUploadResult, error) {
|
||||
return uploadReaderInChunks(ctx, reader, opt, prepareUploadData)
|
||||
}
|
||||
|
||||
func uploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption, prepare func([]byte, *UploadOption) (*preparedUpload, error)) (*ChunkedUploadResult, error) {
|
||||
|
||||
md5Hash := md5.New()
|
||||
var partReader = io.TeeReader(reader, md5Hash)
|
||||
@@ -178,6 +182,23 @@ uploadLoop:
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Hash and prepare only this bounded chunk before minting its volume JWT.
|
||||
chunkMd5 := md5.Sum(buf.Bytes())
|
||||
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
|
||||
var prepared *preparedUpload
|
||||
if opt.UploadFunc == nil {
|
||||
var prepareErr error
|
||||
prepared, prepareErr = prepare(buf.Bytes(), chunkUploadOption(opt))
|
||||
if prepareErr != nil {
|
||||
uploadErrLock.Lock()
|
||||
if uploadErr == nil {
|
||||
uploadErr = fmt.Errorf("prepare chunk: %w", prepareErr)
|
||||
}
|
||||
uploadErrLock.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Assign volume for this chunk
|
||||
_, assignResult, assignErr := opt.AssignFunc(ctx, 1, uint64(size))
|
||||
if assignErr != nil {
|
||||
@@ -196,10 +217,6 @@ uploadLoop:
|
||||
jwt = assignResult.Auth
|
||||
}
|
||||
|
||||
// Calculate MD5 for the chunk
|
||||
chunkMd5 := md5.Sum(buf.Bytes())
|
||||
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
|
||||
|
||||
var uploadResult *UploadResult
|
||||
var uploadResultErr error
|
||||
|
||||
@@ -207,7 +224,7 @@ uploadLoop:
|
||||
// every attempt against the same fid, so ask for a fresh assignment and
|
||||
// retry there rather than losing the whole object to one bad volume.
|
||||
for attempt := 1; ; attempt++ {
|
||||
uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), jwt, chunkMd5B64, opt)
|
||||
uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), prepared, jwt, chunkMd5B64, opt)
|
||||
if uploadResultErr == nil {
|
||||
break
|
||||
}
|
||||
@@ -305,28 +322,23 @@ const chunkAssignAttempts = 3
|
||||
// uploadChunk writes one chunk to its assigned volume. It fans out to every
|
||||
// holder, except for cipher: per-call encryption would give each replica
|
||||
// different bytes, so keep its relay path.
|
||||
func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) {
|
||||
func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) {
|
||||
holders := chunkHolders(assignResult)
|
||||
if opt.UploadFunc == nil && !opt.Cipher && len(holders) > 1 {
|
||||
return uploadChunkToHolders(ctx, holders, assignResult.Fid, data, jwt, md5b64, assignResult.Fsync, opt)
|
||||
}
|
||||
uploadOption := &UploadOption{
|
||||
UploadUrl: fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid),
|
||||
Cipher: opt.Cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: opt.MimeType,
|
||||
PairMap: nil,
|
||||
Jwt: jwt,
|
||||
Md5: md5b64,
|
||||
// One upload call per assignment: the caller retries everything a
|
||||
// same-URL retry would, and a different volume is the better second try.
|
||||
// This bounds retriedUploadData only — doUploadData still reissues once
|
||||
// on a connection reset, with a rewound body, which is a transport
|
||||
// stutter rather than a fresh attempt at the volume. The fan-out path
|
||||
// keeps its retries, where absorbing a blip locally beats cancelling
|
||||
// every holder and re-uploading the chunk.
|
||||
MaxAttempts: 1,
|
||||
return uploadChunkToHolders(ctx, holders, assignResult.Fid, prepared, jwt, md5b64, assignResult.Fsync, opt)
|
||||
}
|
||||
uploadOption := chunkUploadOption(opt)
|
||||
uploadOption.UploadUrl = fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
|
||||
uploadOption.Jwt = jwt
|
||||
uploadOption.Md5 = md5b64
|
||||
// One upload call per assignment: the caller retries everything a
|
||||
// same-URL retry would, and a different volume is the better second try.
|
||||
// This bounds retriedUploadData only — doUploadData still reissues once
|
||||
// on a connection reset, with a rewound body, which is a transport
|
||||
// stutter rather than a fresh attempt at the volume. The fan-out path
|
||||
// keeps its retries, where absorbing a blip locally beats cancelling
|
||||
// every holder and re-uploading the chunk.
|
||||
uploadOption.MaxAttempts = 1
|
||||
if assignResult.Fsync {
|
||||
uploadOption.UploadUrl += "?fsync=true"
|
||||
}
|
||||
@@ -338,7 +350,13 @@ func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, j
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create uploader: %w", err)
|
||||
}
|
||||
return uploader.UploadData(ctx, data, uploadOption)
|
||||
return uploader.retriedUploadData(ctx, prepared, uploadOption)
|
||||
}
|
||||
|
||||
// chunkUploadOption holds the settings that shape a chunk's bytes, shared by
|
||||
// its preparation and every upload of it.
|
||||
func chunkUploadOption(opt *ChunkedUploadOption) *UploadOption {
|
||||
return &UploadOption{Cipher: opt.Cipher, MimeType: opt.MimeType}
|
||||
}
|
||||
|
||||
// chunkHolders returns the assigned volume plus its replica holders.
|
||||
@@ -356,7 +374,7 @@ func chunkHolders(assignResult *AssignResult) []string {
|
||||
// type=replicate). On the first failure it cancels the remaining uploads and
|
||||
// deletes any copies that already landed, so a partial fan-out leaves no
|
||||
// orphaned needle the caller cannot see.
|
||||
func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data []byte, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) {
|
||||
func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) {
|
||||
uploader, err := NewUploader()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create uploader: %w", err)
|
||||
@@ -376,16 +394,11 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data
|
||||
if fsync {
|
||||
uploadUrl += "&fsync=true"
|
||||
}
|
||||
uploadOption := &UploadOption{
|
||||
UploadUrl: uploadUrl,
|
||||
Cipher: false,
|
||||
IsInputCompressed: false,
|
||||
MimeType: opt.MimeType,
|
||||
PairMap: nil,
|
||||
Jwt: jwt,
|
||||
Md5: md5b64,
|
||||
}
|
||||
r, e := uploader.UploadData(ctx, data, uploadOption)
|
||||
uploadOption := chunkUploadOption(opt)
|
||||
uploadOption.UploadUrl = uploadUrl
|
||||
uploadOption.Jwt = jwt
|
||||
uploadOption.Md5 = md5b64
|
||||
r, e := uploader.retriedUploadData(ctx, prepared, uploadOption)
|
||||
outcomes <- outcome{host, r, e}
|
||||
}(host)
|
||||
}
|
||||
|
||||
@@ -278,7 +278,11 @@ func TestUploadChunkToHoldersRollsBackOnPartialFailure(t *testing.T) {
|
||||
defer bad.Close()
|
||||
|
||||
hosts := []string{strings.TrimPrefix(good.URL, "http://"), strings.TrimPrefix(bad.URL, "http://")}
|
||||
_, err := uploadChunkToHolders(context.Background(), hosts, fid, []byte("hello world"), "", "", false, &ChunkedUploadOption{})
|
||||
prepared, err := prepareUploadData([]byte("hello world"), &UploadOption{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = uploadChunkToHolders(context.Background(), hosts, fid, prepared, "", "", false, &ChunkedUploadOption{})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error from a partial fan-out")
|
||||
|
||||
@@ -166,6 +166,8 @@ type HTTPClient interface {
|
||||
// Uploader
|
||||
type Uploader struct {
|
||||
httpClient HTTPClient
|
||||
// prepareData permits deterministic CPU-work timing at the preparation boundary.
|
||||
prepareData func([]byte, *UploadOption) (*preparedUpload, error)
|
||||
// fastUploadHosts records the wire rate of the last sizeable upload to
|
||||
// each host, so a later upload stalling far below it can be blamed on
|
||||
// that one connection rather than the link and get cancelled for a retry
|
||||
@@ -204,6 +206,10 @@ func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader {
|
||||
}
|
||||
|
||||
func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, fsync bool, err error), uploadOption *UploadOption, data []byte) (fileId string, uploadResult *UploadResult, err error) {
|
||||
prepared, err := uploader.prepareUploadData(data, uploadOption)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
doUploadFunc := func() error {
|
||||
var host string
|
||||
var auth security.EncodedJwt
|
||||
@@ -233,7 +239,7 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho
|
||||
}
|
||||
}
|
||||
|
||||
uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)
|
||||
uploadResult, err = uploader.retriedUploadData(context.Background(), prepared, uploadOption)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -313,9 +319,12 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
|
||||
}
|
||||
|
||||
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
|
||||
func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||
uploadResult, err = uploader.retriedUploadData(ctx, data, option)
|
||||
return
|
||||
func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
|
||||
prepared, err := uploader.prepareUploadData(data, option)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return uploader.retriedUploadData(ctx, prepared, option)
|
||||
}
|
||||
|
||||
// Upload sends a POST request to a volume server to upload the content with fast compression
|
||||
@@ -335,11 +344,15 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option
|
||||
return
|
||||
}
|
||||
}
|
||||
uploadResult, uploadErr := uploader.retriedUploadData(ctx, data, option)
|
||||
prepared, err := uploader.prepareUploadData(data, option)
|
||||
if err != nil {
|
||||
return nil, err, data
|
||||
}
|
||||
uploadResult, uploadErr := uploader.retriedUploadData(ctx, prepared, option)
|
||||
return uploadResult, uploadErr, data
|
||||
}
|
||||
|
||||
func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||
func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||
// A stall disarms the watchdog for this upload's remaining attempts only;
|
||||
// keep that on a per-call option so a caller reusing UploadOption starts
|
||||
// the next upload armed.
|
||||
@@ -357,7 +370,7 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op
|
||||
case <-time.After(time.Millisecond * time.Duration(237*(i+1))):
|
||||
}
|
||||
}
|
||||
uploadResult, err = uploader.doUploadData(ctx, data, option)
|
||||
uploadResult, err = uploader.doUploadData(ctx, prepared, option)
|
||||
if err == nil {
|
||||
uploadResult.RetryCount = i
|
||||
return
|
||||
@@ -390,7 +403,19 @@ func isGenericBinaryMimeType(mimeType string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||
// preparedUpload owns the transformed bytes for one buffered chunk. Reusing it
|
||||
// across attempts keeps compression and encryption outside assigned JWT lifetimes.
|
||||
type preparedUpload struct {
|
||||
mimeType string
|
||||
data []byte
|
||||
clearSize int
|
||||
gzip bool
|
||||
cipherKey []byte
|
||||
}
|
||||
|
||||
func prepareUploadData(data []byte, option *UploadOption) (prepared *preparedUpload, err error) {
|
||||
localOption := *option
|
||||
option = &localOption
|
||||
contentIsGzipped := option.IsInputCompressed
|
||||
shouldGzipNow := false
|
||||
if !option.IsInputCompressed && !option.IsReplication {
|
||||
@@ -407,7 +432,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
var compressed []byte
|
||||
compressed, err = util.GzipData(data[0:128])
|
||||
if err != nil {
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
|
||||
}
|
||||
@@ -436,22 +461,35 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
}
|
||||
}
|
||||
|
||||
prepared = &preparedUpload{data: data, clearSize: clearDataLen, gzip: contentIsGzipped, mimeType: option.MimeType}
|
||||
if option.Cipher {
|
||||
// encrypt(gzip(data))
|
||||
|
||||
// encrypt
|
||||
cipherKey := util.GenCipherKey()
|
||||
encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
|
||||
if encryptionErr != nil {
|
||||
err = fmt.Errorf("encrypt input: %w", encryptionErr)
|
||||
return
|
||||
prepared.cipherKey = util.GenCipherKey()
|
||||
prepared.data, err = util.Encrypt(data, prepared.cipherKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encrypt input: %w", err)
|
||||
}
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func (uploader *Uploader) prepareUploadData(data []byte, option *UploadOption) (*preparedUpload, error) {
|
||||
if uploader.prepareData != nil {
|
||||
return uploader.prepareData(data, option)
|
||||
}
|
||||
return prepareUploadData(data, option)
|
||||
}
|
||||
|
||||
func (uploader *Uploader) doUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||
localOption := *option
|
||||
option = &localOption
|
||||
option.MimeType = prepared.mimeType
|
||||
|
||||
if prepared.cipherKey != nil {
|
||||
// upload data
|
||||
uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
|
||||
_, err = w.Write(encryptedData)
|
||||
_, err = w.Write(prepared.data)
|
||||
return
|
||||
}, len(encryptedData), &UploadOption{
|
||||
}, len(prepared.data), &UploadOption{
|
||||
UploadUrl: option.UploadUrl,
|
||||
Filename: "",
|
||||
Cipher: false,
|
||||
@@ -467,24 +505,24 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
}
|
||||
uploadResult.Name = option.Filename
|
||||
uploadResult.Mime = option.MimeType
|
||||
uploadResult.CipherKey = cipherKey
|
||||
uploadResult.CipherKey = prepared.cipherKey
|
||||
// The volume server only ever hashes the ciphertext it stored, so the
|
||||
// chunk ETag has to come from the caller's plaintext digest.
|
||||
uploadResult.ContentMd5 = option.Md5
|
||||
uploadResult.Size = uint32(clearDataLen)
|
||||
if contentIsGzipped {
|
||||
uploadResult.Size = uint32(prepared.clearSize)
|
||||
if prepared.gzip {
|
||||
uploadResult.Gzip = 1
|
||||
}
|
||||
} else {
|
||||
// upload data
|
||||
uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
|
||||
_, err = w.Write(data)
|
||||
_, err = w.Write(prepared.data)
|
||||
return
|
||||
}, len(data), &UploadOption{
|
||||
}, len(prepared.data), &UploadOption{
|
||||
UploadUrl: option.UploadUrl,
|
||||
Filename: option.Filename,
|
||||
Cipher: false,
|
||||
IsInputCompressed: contentIsGzipped,
|
||||
IsInputCompressed: prepared.gzip,
|
||||
IsReplication: option.IsReplication,
|
||||
MimeType: option.MimeType,
|
||||
PairMap: option.PairMap,
|
||||
@@ -496,8 +534,8 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
if uploadResult == nil {
|
||||
return
|
||||
}
|
||||
uploadResult.Size = uint32(clearDataLen)
|
||||
if contentIsGzipped {
|
||||
uploadResult.Size = uint32(prepared.clearSize)
|
||||
if prepared.gzip {
|
||||
uploadResult.Gzip = 1
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user