operation: prepare upload content before assigning the volume (#12714)

* operation: prepare upload content before assigning the volume

* operation: keep the existing upload function names for prepared payloads

UploadData and doUpload prepare once and hand the prepared payload to
retriedUploadData, so the retry loop, doUploadData, uploadChunk and
uploadChunkToHolders take a preparedUpload directly instead of growing
a prepared twin of each and branching on nil.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: send ciphertext whenever the prepared payload carries a cipher key

doUploadData chose the cipher branch from the option, so a payload
prepared under a different option could be sent as plaintext metadata
or lose its key. Decide from the prepared payload itself.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: build a chunk's preparation and upload options in one place

The chunk loop prepared with a hand-written option while uploadChunk and
uploadChunkToHolders built their own, so a setting that shapes the bytes
could reach one and not the other. chunkUploadOption now feeds all three.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: test that a reassigned chunk reuses its prepared payload

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-17 22:36:46 -08:00
1 parent 194e6ede9f
commit deacd575ea
4 files changed
+489 -64

No files matched your search

@@ -0,0 +1,370 @@
package operation
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
jwt "github.com/golang-jwt/jwt/v5"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)
type preparationAssignClient struct {
filer_pb.SeaweedFilerClient
assign func(*filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse
}
func (c *preparationAssignClient) AssignVolume(_ context.Context, req *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) {
return c.assign(req), nil
}
func (c *preparationAssignClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(c)
}
func (c *preparationAssignClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url }
func (c *preparationAssignClient) GetDataCenter() string { return "" }
// The preparation collaborator performs real gzip/encryption and advances only
// the JWT verifier's clock. No wall-clock sleep or slow-machine threshold is used.
func TestUploadPreparesBeforeAssignment(t *testing.T) {
for _, cipher := range []bool{false, true} {
name := "gzip"
if cipher {
name = "gzip_cipher"
}
t.Run(name, func(t *testing.T) {
payload := bytes.Repeat([]byte{0}, 128*1024)
sum := md5.Sum(payload)
digest := base64.StdEncoding.EncodeToString(sum[:])
const fid = "3,01637037d6"
key := []byte("disposable uploader boundary test")
var now atomic.Int64
now.Store(1700000000)
var requests atomic.Int32
var stored []*needle.Needle
var mu sync.Mutex
var parseFailure atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
claims := &security.SeaweedFileIdClaims{}
token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"}))
if err != nil || !token.Valid || claims.Fid != fid {
w.WriteHeader(http.StatusUnauthorized)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
return
}
if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" {
parseFailure.Store(true)
}
n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
if err != nil {
parseFailure.Store(true)
w.WriteHeader(http.StatusBadRequest)
return
}
if cipher {
if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 {
parseFailure.Store(true)
}
} else if gotMd5 != digest {
parseFailure.Store(true)
}
mu.Lock()
stored = append(stored, n)
first := len(stored) == 1
mu.Unlock()
if first {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"error":"busy"}`)
return
}
w.Header().Set("Content-MD5", digest)
w.WriteHeader(http.StatusCreated)
_, _ = io.WriteString(w, `{"name":"long.bin","size":131072}`)
}))
defer server.Close()
uploader := newUploader(server.Client())
preparations := 0
uploader.prepareData = func(data []byte, opt *UploadOption) (*preparedUpload, error) {
preparations++
prepared, err := prepareUploadData(data, opt)
now.Add(11)
return prepared, err
}
assignments := 0
client := &preparationAssignClient{assign: func(req *filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse {
assignments++
if req.ExpectedDataSize != uint64(len(payload)) {
t.Error("assignment did not receive clear size")
}
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key)
if err != nil {
t.Fatal("could not sign disposable fixture")
}
return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: encoded, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}, Fsync: true}
}}
option := &UploadOption{Filename: "long.bin", Cipher: cipher, MimeType: "application/octet-stream", WantMd5: true, PairMap: map[string]string{"test": "private-metadata"}}
gotFid, result, err, clear := uploader.UploadWithRetry(client, &filer_pb.AssignVolumeRequest{Count: 1}, option, bytes.NewReader(payload))
mu.Lock()
saved := append([]*needle.Needle(nil), stored...)
mu.Unlock()
if err != nil {
t.Fatalf("prepared upload was rejected: assignments=%d requests=%d preparations=%d", assignments, requests.Load(), preparations)
}
if gotFid != fid || !bytes.Equal(clear, payload) || result.Size != uint32(len(payload)) || result.Gzip != 1 || result.ContentMd5 != digest {
t.Fatal("clear result metadata changed")
}
if assignments != 1 || preparations != 1 || requests.Load() != 2 || len(saved) != 2 || parseFailure.Load() {
t.Fatalf("outbound/retry contract mismatch: assignments=%d preparations=%d requests=%d", assignments, preparations, requests.Load())
}
if !bytes.Equal(saved[0].Data, saved[1].Data) {
t.Fatal("retry changed prepared bytes")
}
wire := saved[1].Data
if cipher {
if len(result.CipherKey) == 0 {
t.Fatal("missing cipher key")
}
wire, err = util.Decrypt(wire, util.CipherKey(result.CipherKey))
if err != nil {
t.Fatal("cipher payload cannot be decrypted")
}
}
decoded, err := util.DecompressData(wire)
if err != nil || !bytes.Equal(decoded, payload) {
t.Fatal("prepared wire payload changed clear content")
}
if option.MimeType != "application/octet-stream" {
t.Fatal("preparation mutated caller MIME option")
}
})
}
}
func TestUploadChunksPrepareBeforeAssignment(t *testing.T) {
for _, mode := range []string{"relay", "fanout", "cipher"} {
t.Run(mode, func(t *testing.T) {
cipher := mode == "cipher"
payload := bytes.Repeat([]byte{0}, 128*1024)
sum := md5.Sum(payload)
digest := base64.StdEncoding.EncodeToString(sum[:])
const fid = "3,01637037d6"
key := []byte("disposable chunk boundary test")
var now atomic.Int64
now.Store(1700000000)
var posts, deletes, preparations, assignments atomic.Int32
var invalid atomic.Bool
var mu sync.Mutex
var stored []*needle.Needle
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete {
deletes.Add(1)
w.WriteHeader(http.StatusAccepted)
return
}
posts.Add(1)
claims := &security.SeaweedFileIdClaims{}
token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"}))
if err != nil || !token.Valid || claims.Fid != fid {
w.WriteHeader(http.StatusUnauthorized)
_, _ = io.WriteString(w, `{"error":"wrong jwt"}`)
return
}
if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" {
invalid.Store(true)
}
if mode == "fanout" && r.URL.Query().Get("type") != "replicate" {
invalid.Store(true)
}
n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
if err != nil {
invalid.Store(true)
w.WriteHeader(http.StatusBadRequest)
return
}
if cipher {
if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 {
invalid.Store(true)
}
} else if gotMd5 != digest {
invalid.Store(true)
}
mu.Lock()
stored = append(stored, n)
mu.Unlock()
w.Header().Set("Content-MD5", digest)
w.WriteHeader(http.StatusCreated)
_, _ = io.WriteString(w, `{"size":131072}`)
})
primary := httptest.NewServer(handler)
defer primary.Close()
replica := httptest.NewServer(handler)
defer replica.Close()
option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(ctx context.Context, count int, size uint64) (*VolumeAssignRequest, *AssignResult, error) {
assignments.Add(1)
if count != 1 || size != uint64(len(payload)) {
invalid.Store(true)
}
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key)
if err != nil {
return nil, nil, err
}
result := &AssignResult{Fid: fid, Url: strings.TrimPrefix(primary.URL, "http://"), Auth: security.EncodedJwt(encoded), Fsync: true}
if mode == "fanout" {
result.Replicas = []Location{{Url: strings.TrimPrefix(replica.URL, "http://")}}
}
return nil, result, nil
}}
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) {
preparations.Add(1)
prepared, err := prepareUploadData(data, opt)
now.Add(11)
return prepared, err
})
if err != nil {
t.Fatalf("chunk JWT expired during preparation: assignments=%d posts=%d preparations=%d", assignments.Load(), posts.Load(), preparations.Load())
}
wantPosts := int32(1)
if mode == "fanout" {
wantPosts = 2
}
if invalid.Load() || assignments.Load() != 1 || preparations.Load() != 1 || posts.Load() != wantPosts || deletes.Load() != 0 || len(stored) != int(wantPosts) {
t.Fatal("chunk assignment/holder contract changed")
}
if len(result.FileChunks) != 1 || result.TotalSize != int64(len(payload)) || result.FileChunks[0].Size != uint64(len(payload)) || result.FileChunks[0].ETag != digest || !result.FileChunks[0].IsCompressed || result.FileChunks[0].FileId != fid {
t.Fatal("chunk clear metadata changed")
}
if len(stored) == 2 && !bytes.Equal(stored[0].Data, stored[1].Data) {
t.Fatal("holders received different prepared payloads")
}
wire := stored[0].Data
if cipher {
wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey))
if err != nil {
t.Fatal("cannot decrypt chunk")
}
}
decoded, err := util.DecompressData(wire)
if err != nil || !bytes.Equal(decoded, payload) {
t.Fatal("stored chunk differs from clear input")
}
})
}
}
func TestUploadChunksPreserveCustomUploadContract(t *testing.T) {
payload := bytes.Repeat([]byte{0}, 32768)
calls := 0
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), &ChunkedUploadOption{ChunkSize: 65536, Cipher: true, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) {
return nil, &AssignResult{Fid: "3,01637037d6", Url: "unused", Auth: "disposable-token", Fsync: true}, nil
}, UploadFunc: func(_ context.Context, data []byte, opt *UploadOption) (*UploadResult, error) {
calls++
if !bytes.Equal(data, payload) || !opt.Cipher || opt.IsInputCompressed || opt.MimeType != "application/octet-stream" || opt.Jwt != "disposable-token" || opt.UploadUrl != "http://unused/3,01637037d6?fsync=true" || opt.Md5 == "" || opt.MaxAttempts != 1 {
t.Error("custom uploader contract changed")
}
return &UploadResult{Size: uint32(len(data))}, nil
}}, func([]byte, *UploadOption) (*preparedUpload, error) {
t.Fatal("custom uploader must receive original clear bytes without internal preparation")
return nil, nil
})
if err != nil || calls != 1 || len(result.FileChunks) != 1 {
t.Fatal("custom upload did not complete")
}
}
func TestUploadChunksReassignReusesPreparedPayload(t *testing.T) {
for _, cipher := range []bool{false, true} {
t.Run(map[bool]string{false: "relay", true: "cipher"}[cipher], func(t *testing.T) {
payload := bytes.Repeat([]byte{0}, 128*1024)
sum := md5.Sum(payload)
digest := base64.StdEncoding.EncodeToString(sum[:])
fids := []string{"3,01637037d6", "4,02637037d6"}
key := []byte("disposable reassign test")
var mu sync.Mutex
var bodies [][]byte
var postedFids, deletedFids []string
var preparations, assignments atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fid := strings.TrimPrefix(r.URL.Path, "/")
if r.Method == http.MethodDelete {
mu.Lock()
deletedFids = append(deletedFids, fid)
mu.Unlock()
w.WriteHeader(http.StatusAccepted)
return
}
claims := &security.SeaweedFileIdClaims{}
if _, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }); err != nil || claims.Fid != fid {
w.WriteHeader(http.StatusUnauthorized)
return
}
n, _, _, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
mu.Lock()
bodies = append(bodies, n.Data)
postedFids = append(postedFids, fid)
mu.Unlock()
if fid == fids[0] {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"error":"volume unavailable"}`)
return
}
w.Header().Set("Content-MD5", digest)
w.WriteHeader(http.StatusCreated)
_, _ = io.WriteString(w, `{"size":131072}`)
}))
defer server.Close()
option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) {
fid := fids[assignments.Add(1)-1]
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid}).SignedString(key)
if err != nil {
return nil, nil, err
}
return nil, &AssignResult{Fid: fid, Url: strings.TrimPrefix(server.URL, "http://"), Auth: security.EncodedJwt(encoded)}, nil
}}
result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) {
preparations.Add(1)
return prepareUploadData(data, opt)
})
if err != nil {
t.Fatalf("upload after reassignment: %v", err)
}
if preparations.Load() != 1 || assignments.Load() != 2 {
t.Fatalf("preparations=%d assignments=%d, want 1 and 2", preparations.Load(), assignments.Load())
}
if len(bodies) != 2 || !bytes.Equal(bodies[0], bodies[1]) || postedFids[0] != fids[0] || postedFids[1] != fids[1] {
t.Fatalf("posted fids %v; want the same prepared bytes on both assignments", postedFids)
}
if len(deletedFids) != 1 || deletedFids[0] != fids[0] {
t.Fatalf("deleted fids %v, want only the failed %s", deletedFids, fids[0])
}
if len(result.FileChunks) != 1 || result.FileChunks[0].FileId != fids[1] {
t.Fatal("chunk does not name the reassigned fid")
}
wire := bodies[1]
if cipher {
if wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey)); err != nil {
t.Fatalf("decrypt reassigned chunk: %v", err)
}
}
if decoded, err := util.DecompressData(wire); err != nil || !bytes.Equal(decoded, payload) {
t.Fatal("reassigned chunk differs from clear input")
}
})
}
}
+49 -36
View File
@@ -59,6 +59,10 @@ var chunkBufferPool = sync.Pool{
// This prevents OOM by processing the stream in fixed-size chunks
// Returns file chunks, MD5 hash, total size, and any small content stored inline
func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption) (*ChunkedUploadResult, error) {
return uploadReaderInChunks(ctx, reader, opt, prepareUploadData)
}
func uploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption, prepare func([]byte, *UploadOption) (*preparedUpload, error)) (*ChunkedUploadResult, error) {
md5Hash := md5.New()
var partReader = io.TeeReader(reader, md5Hash)
@@ -178,6 +182,23 @@ uploadLoop:
wg.Done()
}()
// Hash and prepare only this bounded chunk before minting its volume JWT.
chunkMd5 := md5.Sum(buf.Bytes())
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
var prepared *preparedUpload
if opt.UploadFunc == nil {
var prepareErr error
prepared, prepareErr = prepare(buf.Bytes(), chunkUploadOption(opt))
if prepareErr != nil {
uploadErrLock.Lock()
if uploadErr == nil {
uploadErr = fmt.Errorf("prepare chunk: %w", prepareErr)
}
uploadErrLock.Unlock()
return
}
}
// Assign volume for this chunk
_, assignResult, assignErr := opt.AssignFunc(ctx, 1, uint64(size))
if assignErr != nil {
@@ -196,10 +217,6 @@ uploadLoop:
jwt = assignResult.Auth
}
// Calculate MD5 for the chunk
chunkMd5 := md5.Sum(buf.Bytes())
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
var uploadResult *UploadResult
var uploadResultErr error
@@ -207,7 +224,7 @@ uploadLoop:
// every attempt against the same fid, so ask for a fresh assignment and
// retry there rather than losing the whole object to one bad volume.
for attempt := 1; ; attempt++ {
uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), jwt, chunkMd5B64, opt)
uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), prepared, jwt, chunkMd5B64, opt)
if uploadResultErr == nil {
break
}
@@ -305,28 +322,23 @@ const chunkAssignAttempts = 3
// uploadChunk writes one chunk to its assigned volume. It fans out to every
// holder, except for cipher: per-call encryption would give each replica
// different bytes, so keep its relay path.
func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) {
func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) {
holders := chunkHolders(assignResult)
if opt.UploadFunc == nil && !opt.Cipher && len(holders) > 1 {
return uploadChunkToHolders(ctx, holders, assignResult.Fid, data, jwt, md5b64, assignResult.Fsync, opt)
}
uploadOption := &UploadOption{
UploadUrl: fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid),
Cipher: opt.Cipher,
IsInputCompressed: false,
MimeType: opt.MimeType,
PairMap: nil,
Jwt: jwt,
Md5: md5b64,
// One upload call per assignment: the caller retries everything a
// same-URL retry would, and a different volume is the better second try.
// This bounds retriedUploadData only — doUploadData still reissues once
// on a connection reset, with a rewound body, which is a transport
// stutter rather than a fresh attempt at the volume. The fan-out path
// keeps its retries, where absorbing a blip locally beats cancelling
// every holder and re-uploading the chunk.
MaxAttempts: 1,
return uploadChunkToHolders(ctx, holders, assignResult.Fid, prepared, jwt, md5b64, assignResult.Fsync, opt)
}
uploadOption := chunkUploadOption(opt)
uploadOption.UploadUrl = fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
uploadOption.Jwt = jwt
uploadOption.Md5 = md5b64
// One upload call per assignment: the caller retries everything a
// same-URL retry would, and a different volume is the better second try.
// This bounds retriedUploadData only — doUploadData still reissues once
// on a connection reset, with a rewound body, which is a transport
// stutter rather than a fresh attempt at the volume. The fan-out path
// keeps its retries, where absorbing a blip locally beats cancelling
// every holder and re-uploading the chunk.
uploadOption.MaxAttempts = 1
if assignResult.Fsync {
uploadOption.UploadUrl += "?fsync=true"
}
@@ -338,7 +350,13 @@ func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, j
if err != nil {
return nil, fmt.Errorf("create uploader: %w", err)
}
return uploader.UploadData(ctx, data, uploadOption)
return uploader.retriedUploadData(ctx, prepared, uploadOption)
}
// chunkUploadOption holds the settings that shape a chunk's bytes, shared by
// its preparation and every upload of it.
func chunkUploadOption(opt *ChunkedUploadOption) *UploadOption {
return &UploadOption{Cipher: opt.Cipher, MimeType: opt.MimeType}
}
// chunkHolders returns the assigned volume plus its replica holders.
@@ -356,7 +374,7 @@ func chunkHolders(assignResult *AssignResult) []string {
// type=replicate). On the first failure it cancels the remaining uploads and
// deletes any copies that already landed, so a partial fan-out leaves no
// orphaned needle the caller cannot see.
func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data []byte, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) {
func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) {
uploader, err := NewUploader()
if err != nil {
return nil, fmt.Errorf("create uploader: %w", err)
@@ -376,16 +394,11 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data
if fsync {
uploadUrl += "&fsync=true"
}
uploadOption := &UploadOption{
UploadUrl: uploadUrl,
Cipher: false,
IsInputCompressed: false,
MimeType: opt.MimeType,
PairMap: nil,
Jwt: jwt,
Md5: md5b64,
}
r, e := uploader.UploadData(ctx, data, uploadOption)
uploadOption := chunkUploadOption(opt)
uploadOption.UploadUrl = uploadUrl
uploadOption.Jwt = jwt
uploadOption.Md5 = md5b64
r, e := uploader.retriedUploadData(ctx, prepared, uploadOption)
outcomes <- outcome{host, r, e}
}(host)
}
+5 -1
View File
@@ -278,7 +278,11 @@ func TestUploadChunkToHoldersRollsBackOnPartialFailure(t *testing.T) {
defer bad.Close()
hosts := []string{strings.TrimPrefix(good.URL, "http://"), strings.TrimPrefix(bad.URL, "http://")}
_, err := uploadChunkToHolders(context.Background(), hosts, fid, []byte("hello world"), "", "", false, &ChunkedUploadOption{})
prepared, err := prepareUploadData([]byte("hello world"), &UploadOption{})
if err != nil {
t.Fatal(err)
}
_, err = uploadChunkToHolders(context.Background(), hosts, fid, prepared, "", "", false, &ChunkedUploadOption{})
if err == nil {
t.Fatal("expected error from a partial fan-out")
+65 -27
View File
@@ -166,6 +166,8 @@ type HTTPClient interface {
// Uploader
type Uploader struct {
httpClient HTTPClient
// prepareData permits deterministic CPU-work timing at the preparation boundary.
prepareData func([]byte, *UploadOption) (*preparedUpload, error)
// fastUploadHosts records the wire rate of the last sizeable upload to
// each host, so a later upload stalling far below it can be blamed on
// that one connection rather than the link and get cancelled for a retry
@@ -204,6 +206,10 @@ func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader {
}
func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, fsync bool, err error), uploadOption *UploadOption, data []byte) (fileId string, uploadResult *UploadResult, err error) {
prepared, err := uploader.prepareUploadData(data, uploadOption)
if err != nil {
return "", nil, err
}
doUploadFunc := func() error {
var host string
var auth security.EncodedJwt
@@ -233,7 +239,7 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho
}
}
uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)
uploadResult, err = uploader.retriedUploadData(context.Background(), prepared, uploadOption)
return err
}
@@ -313,9 +319,12 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
}
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
uploadResult, err = uploader.retriedUploadData(ctx, data, option)
return
func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
prepared, err := uploader.prepareUploadData(data, option)
if err != nil {
return nil, err
}
return uploader.retriedUploadData(ctx, prepared, option)
}
// Upload sends a POST request to a volume server to upload the content with fast compression
@@ -335,11 +344,15 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option
return
}
}
uploadResult, uploadErr := uploader.retriedUploadData(ctx, data, option)
prepared, err := uploader.prepareUploadData(data, option)
if err != nil {
return nil, err, data
}
uploadResult, uploadErr := uploader.retriedUploadData(ctx, prepared, option)
return uploadResult, uploadErr, data
}
func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) {
// A stall disarms the watchdog for this upload's remaining attempts only;
// keep that on a per-call option so a caller reusing UploadOption starts
// the next upload armed.
@@ -357,7 +370,7 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op
case <-time.After(time.Millisecond * time.Duration(237*(i+1))):
}
}
uploadResult, err = uploader.doUploadData(ctx, data, option)
uploadResult, err = uploader.doUploadData(ctx, prepared, option)
if err == nil {
uploadResult.RetryCount = i
return
@@ -390,7 +403,19 @@ func isGenericBinaryMimeType(mimeType string) bool {
return false
}
func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
// preparedUpload owns the transformed bytes for one buffered chunk. Reusing it
// across attempts keeps compression and encryption outside assigned JWT lifetimes.
type preparedUpload struct {
mimeType string
data []byte
clearSize int
gzip bool
cipherKey []byte
}
func prepareUploadData(data []byte, option *UploadOption) (prepared *preparedUpload, err error) {
localOption := *option
option = &localOption
contentIsGzipped := option.IsInputCompressed
shouldGzipNow := false
if !option.IsInputCompressed && !option.IsReplication {
@@ -407,7 +432,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
var compressed []byte
compressed, err = util.GzipData(data[0:128])
if err != nil {
return
return nil, err
}
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
}
@@ -436,22 +461,35 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
}
}
prepared = &preparedUpload{data: data, clearSize: clearDataLen, gzip: contentIsGzipped, mimeType: option.MimeType}
if option.Cipher {
// encrypt(gzip(data))
// encrypt
cipherKey := util.GenCipherKey()
encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
if encryptionErr != nil {
err = fmt.Errorf("encrypt input: %w", encryptionErr)
return
prepared.cipherKey = util.GenCipherKey()
prepared.data, err = util.Encrypt(data, prepared.cipherKey)
if err != nil {
return nil, fmt.Errorf("encrypt input: %w", err)
}
}
return prepared, nil
}
func (uploader *Uploader) prepareUploadData(data []byte, option *UploadOption) (*preparedUpload, error) {
if uploader.prepareData != nil {
return uploader.prepareData(data, option)
}
return prepareUploadData(data, option)
}
func (uploader *Uploader) doUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) {
localOption := *option
option = &localOption
option.MimeType = prepared.mimeType
if prepared.cipherKey != nil {
// upload data
uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
_, err = w.Write(encryptedData)
_, err = w.Write(prepared.data)
return
}, len(encryptedData), &UploadOption{
}, len(prepared.data), &UploadOption{
UploadUrl: option.UploadUrl,
Filename: "",
Cipher: false,
@@ -467,24 +505,24 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
}
uploadResult.Name = option.Filename
uploadResult.Mime = option.MimeType
uploadResult.CipherKey = cipherKey
uploadResult.CipherKey = prepared.cipherKey
// The volume server only ever hashes the ciphertext it stored, so the
// chunk ETag has to come from the caller's plaintext digest.
uploadResult.ContentMd5 = option.Md5
uploadResult.Size = uint32(clearDataLen)
if contentIsGzipped {
uploadResult.Size = uint32(prepared.clearSize)
if prepared.gzip {
uploadResult.Gzip = 1
}
} else {
// upload data
uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
_, err = w.Write(data)
_, err = w.Write(prepared.data)
return
}, len(data), &UploadOption{
}, len(prepared.data), &UploadOption{
UploadUrl: option.UploadUrl,
Filename: option.Filename,
Cipher: false,
IsInputCompressed: contentIsGzipped,
IsInputCompressed: prepared.gzip,
IsReplication: option.IsReplication,
MimeType: option.MimeType,
PairMap: option.PairMap,
@@ -496,8 +534,8 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
if uploadResult == nil {
return
}
uploadResult.Size = uint32(clearDataLen)
if contentIsGzipped {
uploadResult.Size = uint32(prepared.clearSize)
if prepared.gzip {
uploadResult.Gzip = 1
}
}