diff --git a/weed/operation/upload_auth_preparation_test.go b/weed/operation/upload_auth_preparation_test.go new file mode 100644 index 0000000000..825a4c7a8c --- /dev/null +++ b/weed/operation/upload_auth_preparation_test.go @@ -0,0 +1,370 @@ +package operation + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc" +) + +type preparationAssignClient struct { + filer_pb.SeaweedFilerClient + assign func(*filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse +} + +func (c *preparationAssignClient) AssignVolume(_ context.Context, req *filer_pb.AssignVolumeRequest, _ ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) { + return c.assign(req), nil +} +func (c *preparationAssignClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error { + return fn(c) +} +func (c *preparationAssignClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url } +func (c *preparationAssignClient) GetDataCenter() string { return "" } + +// The preparation collaborator performs real gzip/encryption and advances only +// the JWT verifier's clock. No wall-clock sleep or slow-machine threshold is used. +func TestUploadPreparesBeforeAssignment(t *testing.T) { + for _, cipher := range []bool{false, true} { + name := "gzip" + if cipher { + name = "gzip_cipher" + } + t.Run(name, func(t *testing.T) { + payload := bytes.Repeat([]byte{0}, 128*1024) + sum := md5.Sum(payload) + digest := base64.StdEncoding.EncodeToString(sum[:]) + const fid = "3,01637037d6" + key := []byte("disposable uploader boundary test") + var now atomic.Int64 + now.Store(1700000000) + var requests atomic.Int32 + var stored []*needle.Needle + var mu sync.Mutex + var parseFailure atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + claims := &security.SeaweedFileIdClaims{} + token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"})) + if err != nil || !token.Valid || claims.Fid != fid { + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" { + parseFailure.Store(true) + } + n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{}) + if err != nil { + parseFailure.Store(true) + w.WriteHeader(http.StatusBadRequest) + return + } + if cipher { + if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 { + parseFailure.Store(true) + } + } else if gotMd5 != digest { + parseFailure.Store(true) + } + mu.Lock() + stored = append(stored, n) + first := len(stored) == 1 + mu.Unlock() + if first { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"error":"busy"}`) + return + } + w.Header().Set("Content-MD5", digest) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{"name":"long.bin","size":131072}`) + })) + defer server.Close() + uploader := newUploader(server.Client()) + preparations := 0 + uploader.prepareData = func(data []byte, opt *UploadOption) (*preparedUpload, error) { + preparations++ + prepared, err := prepareUploadData(data, opt) + now.Add(11) + return prepared, err + } + assignments := 0 + client := &preparationAssignClient{assign: func(req *filer_pb.AssignVolumeRequest) *filer_pb.AssignVolumeResponse { + assignments++ + if req.ExpectedDataSize != uint64(len(payload)) { + t.Error("assignment did not receive clear size") + } + encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key) + if err != nil { + t.Fatal("could not sign disposable fixture") + } + return &filer_pb.AssignVolumeResponse{FileId: fid, Auth: encoded, Location: &filer_pb.Location{Url: strings.TrimPrefix(server.URL, "http://")}, Fsync: true} + }} + option := &UploadOption{Filename: "long.bin", Cipher: cipher, MimeType: "application/octet-stream", WantMd5: true, PairMap: map[string]string{"test": "private-metadata"}} + gotFid, result, err, clear := uploader.UploadWithRetry(client, &filer_pb.AssignVolumeRequest{Count: 1}, option, bytes.NewReader(payload)) + mu.Lock() + saved := append([]*needle.Needle(nil), stored...) + mu.Unlock() + if err != nil { + t.Fatalf("prepared upload was rejected: assignments=%d requests=%d preparations=%d", assignments, requests.Load(), preparations) + } + if gotFid != fid || !bytes.Equal(clear, payload) || result.Size != uint32(len(payload)) || result.Gzip != 1 || result.ContentMd5 != digest { + t.Fatal("clear result metadata changed") + } + if assignments != 1 || preparations != 1 || requests.Load() != 2 || len(saved) != 2 || parseFailure.Load() { + t.Fatalf("outbound/retry contract mismatch: assignments=%d preparations=%d requests=%d", assignments, preparations, requests.Load()) + } + if !bytes.Equal(saved[0].Data, saved[1].Data) { + t.Fatal("retry changed prepared bytes") + } + wire := saved[1].Data + if cipher { + if len(result.CipherKey) == 0 { + t.Fatal("missing cipher key") + } + wire, err = util.Decrypt(wire, util.CipherKey(result.CipherKey)) + if err != nil { + t.Fatal("cipher payload cannot be decrypted") + } + } + decoded, err := util.DecompressData(wire) + if err != nil || !bytes.Equal(decoded, payload) { + t.Fatal("prepared wire payload changed clear content") + } + if option.MimeType != "application/octet-stream" { + t.Fatal("preparation mutated caller MIME option") + } + }) + } +} + +func TestUploadChunksPrepareBeforeAssignment(t *testing.T) { + for _, mode := range []string{"relay", "fanout", "cipher"} { + t.Run(mode, func(t *testing.T) { + cipher := mode == "cipher" + payload := bytes.Repeat([]byte{0}, 128*1024) + sum := md5.Sum(payload) + digest := base64.StdEncoding.EncodeToString(sum[:]) + const fid = "3,01637037d6" + key := []byte("disposable chunk boundary test") + var now atomic.Int64 + now.Store(1700000000) + var posts, deletes, preparations, assignments atomic.Int32 + var invalid atomic.Bool + var mu sync.Mutex + var stored []*needle.Needle + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + deletes.Add(1) + w.WriteHeader(http.StatusAccepted) + return + } + posts.Add(1) + claims := &security.SeaweedFileIdClaims{} + token, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }, jwt.WithTimeFunc(func() time.Time { return time.Unix(now.Load(), 0) }), jwt.WithValidMethods([]string{"HS256"})) + if err != nil || !token.Valid || claims.Fid != fid { + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, `{"error":"wrong jwt"}`) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/"+fid || r.URL.Query().Get("fsync") != "true" { + invalid.Store(true) + } + if mode == "fanout" && r.URL.Query().Get("type") != "replicate" { + invalid.Store(true) + } + n, _, gotMd5, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{}) + if err != nil { + invalid.Store(true) + w.WriteHeader(http.StatusBadRequest) + return + } + if cipher { + if gotMd5 != "" || len(n.Name) != 0 || len(n.Mime) != 0 || len(n.Pairs) != 0 { + invalid.Store(true) + } + } else if gotMd5 != digest { + invalid.Store(true) + } + mu.Lock() + stored = append(stored, n) + mu.Unlock() + w.Header().Set("Content-MD5", digest) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{"size":131072}`) + }) + primary := httptest.NewServer(handler) + defer primary.Close() + replica := httptest.NewServer(handler) + defer replica.Close() + option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(ctx context.Context, count int, size uint64) (*VolumeAssignRequest, *AssignResult, error) { + assignments.Add(1) + if count != 1 || size != uint64(len(payload)) { + invalid.Store(true) + } + encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Unix(now.Load()+10, 0))}}).SignedString(key) + if err != nil { + return nil, nil, err + } + result := &AssignResult{Fid: fid, Url: strings.TrimPrefix(primary.URL, "http://"), Auth: security.EncodedJwt(encoded), Fsync: true} + if mode == "fanout" { + result.Replicas = []Location{{Url: strings.TrimPrefix(replica.URL, "http://")}} + } + return nil, result, nil + }} + result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) { + preparations.Add(1) + prepared, err := prepareUploadData(data, opt) + now.Add(11) + return prepared, err + }) + if err != nil { + t.Fatalf("chunk JWT expired during preparation: assignments=%d posts=%d preparations=%d", assignments.Load(), posts.Load(), preparations.Load()) + } + wantPosts := int32(1) + if mode == "fanout" { + wantPosts = 2 + } + if invalid.Load() || assignments.Load() != 1 || preparations.Load() != 1 || posts.Load() != wantPosts || deletes.Load() != 0 || len(stored) != int(wantPosts) { + t.Fatal("chunk assignment/holder contract changed") + } + if len(result.FileChunks) != 1 || result.TotalSize != int64(len(payload)) || result.FileChunks[0].Size != uint64(len(payload)) || result.FileChunks[0].ETag != digest || !result.FileChunks[0].IsCompressed || result.FileChunks[0].FileId != fid { + t.Fatal("chunk clear metadata changed") + } + if len(stored) == 2 && !bytes.Equal(stored[0].Data, stored[1].Data) { + t.Fatal("holders received different prepared payloads") + } + wire := stored[0].Data + if cipher { + wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey)) + if err != nil { + t.Fatal("cannot decrypt chunk") + } + } + decoded, err := util.DecompressData(wire) + if err != nil || !bytes.Equal(decoded, payload) { + t.Fatal("stored chunk differs from clear input") + } + }) + } +} + +func TestUploadChunksPreserveCustomUploadContract(t *testing.T) { + payload := bytes.Repeat([]byte{0}, 32768) + calls := 0 + result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), &ChunkedUploadOption{ChunkSize: 65536, Cipher: true, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) { + return nil, &AssignResult{Fid: "3,01637037d6", Url: "unused", Auth: "disposable-token", Fsync: true}, nil + }, UploadFunc: func(_ context.Context, data []byte, opt *UploadOption) (*UploadResult, error) { + calls++ + if !bytes.Equal(data, payload) || !opt.Cipher || opt.IsInputCompressed || opt.MimeType != "application/octet-stream" || opt.Jwt != "disposable-token" || opt.UploadUrl != "http://unused/3,01637037d6?fsync=true" || opt.Md5 == "" || opt.MaxAttempts != 1 { + t.Error("custom uploader contract changed") + } + return &UploadResult{Size: uint32(len(data))}, nil + }}, func([]byte, *UploadOption) (*preparedUpload, error) { + t.Fatal("custom uploader must receive original clear bytes without internal preparation") + return nil, nil + }) + if err != nil || calls != 1 || len(result.FileChunks) != 1 { + t.Fatal("custom upload did not complete") + } +} + +func TestUploadChunksReassignReusesPreparedPayload(t *testing.T) { + for _, cipher := range []bool{false, true} { + t.Run(map[bool]string{false: "relay", true: "cipher"}[cipher], func(t *testing.T) { + payload := bytes.Repeat([]byte{0}, 128*1024) + sum := md5.Sum(payload) + digest := base64.StdEncoding.EncodeToString(sum[:]) + fids := []string{"3,01637037d6", "4,02637037d6"} + key := []byte("disposable reassign test") + var mu sync.Mutex + var bodies [][]byte + var postedFids, deletedFids []string + var preparations, assignments atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fid := strings.TrimPrefix(r.URL.Path, "/") + if r.Method == http.MethodDelete { + mu.Lock() + deletedFids = append(deletedFids, fid) + mu.Unlock() + w.WriteHeader(http.StatusAccepted) + return + } + claims := &security.SeaweedFileIdClaims{} + if _, err := jwt.ParseWithClaims(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), claims, func(*jwt.Token) (interface{}, error) { return key, nil }); err != nil || claims.Fid != fid { + w.WriteHeader(http.StatusUnauthorized) + return + } + n, _, _, err := needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{}) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + mu.Lock() + bodies = append(bodies, n.Data) + postedFids = append(postedFids, fid) + mu.Unlock() + if fid == fids[0] { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = io.WriteString(w, `{"error":"volume unavailable"}`) + return + } + w.Header().Set("Content-MD5", digest) + w.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(w, `{"size":131072}`) + })) + defer server.Close() + option := &ChunkedUploadOption{ChunkSize: 256 * 1024, Cipher: cipher, MimeType: "application/octet-stream", AssignFunc: func(context.Context, int, uint64) (*VolumeAssignRequest, *AssignResult, error) { + fid := fids[assignments.Add(1)-1] + encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, security.SeaweedFileIdClaims{Fid: fid}).SignedString(key) + if err != nil { + return nil, nil, err + } + return nil, &AssignResult{Fid: fid, Url: strings.TrimPrefix(server.URL, "http://"), Auth: security.EncodedJwt(encoded)}, nil + }} + result, err := uploadReaderInChunks(context.Background(), bytes.NewReader(payload), option, func(data []byte, opt *UploadOption) (*preparedUpload, error) { + preparations.Add(1) + return prepareUploadData(data, opt) + }) + if err != nil { + t.Fatalf("upload after reassignment: %v", err) + } + if preparations.Load() != 1 || assignments.Load() != 2 { + t.Fatalf("preparations=%d assignments=%d, want 1 and 2", preparations.Load(), assignments.Load()) + } + if len(bodies) != 2 || !bytes.Equal(bodies[0], bodies[1]) || postedFids[0] != fids[0] || postedFids[1] != fids[1] { + t.Fatalf("posted fids %v; want the same prepared bytes on both assignments", postedFids) + } + if len(deletedFids) != 1 || deletedFids[0] != fids[0] { + t.Fatalf("deleted fids %v, want only the failed %s", deletedFids, fids[0]) + } + if len(result.FileChunks) != 1 || result.FileChunks[0].FileId != fids[1] { + t.Fatal("chunk does not name the reassigned fid") + } + wire := bodies[1] + if cipher { + if wire, err = util.Decrypt(wire, util.CipherKey(result.FileChunks[0].CipherKey)); err != nil { + t.Fatalf("decrypt reassigned chunk: %v", err) + } + } + if decoded, err := util.DecompressData(wire); err != nil || !bytes.Equal(decoded, payload) { + t.Fatal("reassigned chunk differs from clear input") + } + }) + } +} diff --git a/weed/operation/upload_chunked.go b/weed/operation/upload_chunked.go index 8124500dfa..235c8457f1 100644 --- a/weed/operation/upload_chunked.go +++ b/weed/operation/upload_chunked.go @@ -59,6 +59,10 @@ var chunkBufferPool = sync.Pool{ // This prevents OOM by processing the stream in fixed-size chunks // Returns file chunks, MD5 hash, total size, and any small content stored inline func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption) (*ChunkedUploadResult, error) { + return uploadReaderInChunks(ctx, reader, opt, prepareUploadData) +} + +func uploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption, prepare func([]byte, *UploadOption) (*preparedUpload, error)) (*ChunkedUploadResult, error) { md5Hash := md5.New() var partReader = io.TeeReader(reader, md5Hash) @@ -178,6 +182,23 @@ uploadLoop: wg.Done() }() + // Hash and prepare only this bounded chunk before minting its volume JWT. + chunkMd5 := md5.Sum(buf.Bytes()) + chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:]) + var prepared *preparedUpload + if opt.UploadFunc == nil { + var prepareErr error + prepared, prepareErr = prepare(buf.Bytes(), chunkUploadOption(opt)) + if prepareErr != nil { + uploadErrLock.Lock() + if uploadErr == nil { + uploadErr = fmt.Errorf("prepare chunk: %w", prepareErr) + } + uploadErrLock.Unlock() + return + } + } + // Assign volume for this chunk _, assignResult, assignErr := opt.AssignFunc(ctx, 1, uint64(size)) if assignErr != nil { @@ -196,10 +217,6 @@ uploadLoop: jwt = assignResult.Auth } - // Calculate MD5 for the chunk - chunkMd5 := md5.Sum(buf.Bytes()) - chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:]) - var uploadResult *UploadResult var uploadResultErr error @@ -207,7 +224,7 @@ uploadLoop: // every attempt against the same fid, so ask for a fresh assignment and // retry there rather than losing the whole object to one bad volume. for attempt := 1; ; attempt++ { - uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), jwt, chunkMd5B64, opt) + uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), prepared, jwt, chunkMd5B64, opt) if uploadResultErr == nil { break } @@ -305,28 +322,23 @@ const chunkAssignAttempts = 3 // uploadChunk writes one chunk to its assigned volume. It fans out to every // holder, except for cipher: per-call encryption would give each replica // different bytes, so keep its relay path. -func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) { +func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) { holders := chunkHolders(assignResult) if opt.UploadFunc == nil && !opt.Cipher && len(holders) > 1 { - return uploadChunkToHolders(ctx, holders, assignResult.Fid, data, jwt, md5b64, assignResult.Fsync, opt) - } - uploadOption := &UploadOption{ - UploadUrl: fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid), - Cipher: opt.Cipher, - IsInputCompressed: false, - MimeType: opt.MimeType, - PairMap: nil, - Jwt: jwt, - Md5: md5b64, - // One upload call per assignment: the caller retries everything a - // same-URL retry would, and a different volume is the better second try. - // This bounds retriedUploadData only — doUploadData still reissues once - // on a connection reset, with a rewound body, which is a transport - // stutter rather than a fresh attempt at the volume. The fan-out path - // keeps its retries, where absorbing a blip locally beats cancelling - // every holder and re-uploading the chunk. - MaxAttempts: 1, + return uploadChunkToHolders(ctx, holders, assignResult.Fid, prepared, jwt, md5b64, assignResult.Fsync, opt) } + uploadOption := chunkUploadOption(opt) + uploadOption.UploadUrl = fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) + uploadOption.Jwt = jwt + uploadOption.Md5 = md5b64 + // One upload call per assignment: the caller retries everything a + // same-URL retry would, and a different volume is the better second try. + // This bounds retriedUploadData only — doUploadData still reissues once + // on a connection reset, with a rewound body, which is a transport + // stutter rather than a fresh attempt at the volume. The fan-out path + // keeps its retries, where absorbing a blip locally beats cancelling + // every holder and re-uploading the chunk. + uploadOption.MaxAttempts = 1 if assignResult.Fsync { uploadOption.UploadUrl += "?fsync=true" } @@ -338,7 +350,13 @@ func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, j if err != nil { return nil, fmt.Errorf("create uploader: %w", err) } - return uploader.UploadData(ctx, data, uploadOption) + return uploader.retriedUploadData(ctx, prepared, uploadOption) +} + +// chunkUploadOption holds the settings that shape a chunk's bytes, shared by +// its preparation and every upload of it. +func chunkUploadOption(opt *ChunkedUploadOption) *UploadOption { + return &UploadOption{Cipher: opt.Cipher, MimeType: opt.MimeType} } // chunkHolders returns the assigned volume plus its replica holders. @@ -356,7 +374,7 @@ func chunkHolders(assignResult *AssignResult) []string { // type=replicate). On the first failure it cancels the remaining uploads and // deletes any copies that already landed, so a partial fan-out leaves no // orphaned needle the caller cannot see. -func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data []byte, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) { +func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, prepared *preparedUpload, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) { uploader, err := NewUploader() if err != nil { return nil, fmt.Errorf("create uploader: %w", err) @@ -376,16 +394,11 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data if fsync { uploadUrl += "&fsync=true" } - uploadOption := &UploadOption{ - UploadUrl: uploadUrl, - Cipher: false, - IsInputCompressed: false, - MimeType: opt.MimeType, - PairMap: nil, - Jwt: jwt, - Md5: md5b64, - } - r, e := uploader.UploadData(ctx, data, uploadOption) + uploadOption := chunkUploadOption(opt) + uploadOption.UploadUrl = uploadUrl + uploadOption.Jwt = jwt + uploadOption.Md5 = md5b64 + r, e := uploader.retriedUploadData(ctx, prepared, uploadOption) outcomes <- outcome{host, r, e} }(host) } diff --git a/weed/operation/upload_chunked_test.go b/weed/operation/upload_chunked_test.go index 87ab218ebc..cd54b0489f 100644 --- a/weed/operation/upload_chunked_test.go +++ b/weed/operation/upload_chunked_test.go @@ -278,7 +278,11 @@ func TestUploadChunkToHoldersRollsBackOnPartialFailure(t *testing.T) { defer bad.Close() hosts := []string{strings.TrimPrefix(good.URL, "http://"), strings.TrimPrefix(bad.URL, "http://")} - _, err := uploadChunkToHolders(context.Background(), hosts, fid, []byte("hello world"), "", "", false, &ChunkedUploadOption{}) + prepared, err := prepareUploadData([]byte("hello world"), &UploadOption{}) + if err != nil { + t.Fatal(err) + } + _, err = uploadChunkToHolders(context.Background(), hosts, fid, prepared, "", "", false, &ChunkedUploadOption{}) if err == nil { t.Fatal("expected error from a partial fan-out") diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 8a0544bcae..0b26dc192e 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -166,6 +166,8 @@ type HTTPClient interface { // Uploader type Uploader struct { httpClient HTTPClient + // prepareData permits deterministic CPU-work timing at the preparation boundary. + prepareData func([]byte, *UploadOption) (*preparedUpload, error) // fastUploadHosts records the wire rate of the last sizeable upload to // each host, so a later upload stalling far below it can be blamed on // that one connection rather than the link and get cancelled for a retry @@ -204,6 +206,10 @@ func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader { } func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, fsync bool, err error), uploadOption *UploadOption, data []byte) (fileId string, uploadResult *UploadResult, err error) { + prepared, err := uploader.prepareUploadData(data, uploadOption) + if err != nil { + return "", nil, err + } doUploadFunc := func() error { var host string var auth security.EncodedJwt @@ -233,7 +239,7 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho } } - uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption) + uploadResult, err = uploader.retriedUploadData(context.Background(), prepared, uploadOption) return err } @@ -313,9 +319,12 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi } // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { - uploadResult, err = uploader.retriedUploadData(ctx, data, option) - return +func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) { + prepared, err := uploader.prepareUploadData(data, option) + if err != nil { + return nil, err + } + return uploader.retriedUploadData(ctx, prepared, option) } // Upload sends a POST request to a volume server to upload the content with fast compression @@ -335,11 +344,15 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option return } } - uploadResult, uploadErr := uploader.retriedUploadData(ctx, data, option) + prepared, err := uploader.prepareUploadData(data, option) + if err != nil { + return nil, err, data + } + uploadResult, uploadErr := uploader.retriedUploadData(ctx, prepared, option) return uploadResult, uploadErr, data } -func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { +func (uploader *Uploader) retriedUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) { // A stall disarms the watchdog for this upload's remaining attempts only; // keep that on a per-call option so a caller reusing UploadOption starts // the next upload armed. @@ -357,7 +370,7 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op case <-time.After(time.Millisecond * time.Duration(237*(i+1))): } } - uploadResult, err = uploader.doUploadData(ctx, data, option) + uploadResult, err = uploader.doUploadData(ctx, prepared, option) if err == nil { uploadResult.RetryCount = i return @@ -390,7 +403,19 @@ func isGenericBinaryMimeType(mimeType string) bool { return false } -func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { +// preparedUpload owns the transformed bytes for one buffered chunk. Reusing it +// across attempts keeps compression and encryption outside assigned JWT lifetimes. +type preparedUpload struct { + mimeType string + data []byte + clearSize int + gzip bool + cipherKey []byte +} + +func prepareUploadData(data []byte, option *UploadOption) (prepared *preparedUpload, err error) { + localOption := *option + option = &localOption contentIsGzipped := option.IsInputCompressed shouldGzipNow := false if !option.IsInputCompressed && !option.IsReplication { @@ -407,7 +432,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option var compressed []byte compressed, err = util.GzipData(data[0:128]) if err != nil { - return + return nil, err } shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% } @@ -436,22 +461,35 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option } } + prepared = &preparedUpload{data: data, clearSize: clearDataLen, gzip: contentIsGzipped, mimeType: option.MimeType} if option.Cipher { - // encrypt(gzip(data)) - - // encrypt - cipherKey := util.GenCipherKey() - encryptedData, encryptionErr := util.Encrypt(data, cipherKey) - if encryptionErr != nil { - err = fmt.Errorf("encrypt input: %w", encryptionErr) - return + prepared.cipherKey = util.GenCipherKey() + prepared.data, err = util.Encrypt(data, prepared.cipherKey) + if err != nil { + return nil, fmt.Errorf("encrypt input: %w", err) } + } + return prepared, nil +} +func (uploader *Uploader) prepareUploadData(data []byte, option *UploadOption) (*preparedUpload, error) { + if uploader.prepareData != nil { + return uploader.prepareData(data, option) + } + return prepareUploadData(data, option) +} + +func (uploader *Uploader) doUploadData(ctx context.Context, prepared *preparedUpload, option *UploadOption) (uploadResult *UploadResult, err error) { + localOption := *option + option = &localOption + option.MimeType = prepared.mimeType + + if prepared.cipherKey != nil { // upload data uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) { - _, err = w.Write(encryptedData) + _, err = w.Write(prepared.data) return - }, len(encryptedData), &UploadOption{ + }, len(prepared.data), &UploadOption{ UploadUrl: option.UploadUrl, Filename: "", Cipher: false, @@ -467,24 +505,24 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option } uploadResult.Name = option.Filename uploadResult.Mime = option.MimeType - uploadResult.CipherKey = cipherKey + uploadResult.CipherKey = prepared.cipherKey // The volume server only ever hashes the ciphertext it stored, so the // chunk ETag has to come from the caller's plaintext digest. uploadResult.ContentMd5 = option.Md5 - uploadResult.Size = uint32(clearDataLen) - if contentIsGzipped { + uploadResult.Size = uint32(prepared.clearSize) + if prepared.gzip { uploadResult.Gzip = 1 } } else { // upload data uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) { - _, err = w.Write(data) + _, err = w.Write(prepared.data) return - }, len(data), &UploadOption{ + }, len(prepared.data), &UploadOption{ UploadUrl: option.UploadUrl, Filename: option.Filename, Cipher: false, - IsInputCompressed: contentIsGzipped, + IsInputCompressed: prepared.gzip, IsReplication: option.IsReplication, MimeType: option.MimeType, PairMap: option.PairMap, @@ -496,8 +534,8 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option if uploadResult == nil { return } - uploadResult.Size = uint32(clearDataLen) - if contentIsGzipped { + uploadResult.Size = uint32(prepared.clearSize) + if prepared.gzip { uploadResult.Gzip = 1 } }