Compare commits

...
Author SHA1 Message Date
Devin 9d84ecca2f upload -dir: finish every file before reporting a failed upload
A failed batch stopped all remaining batches, and the file-list path
returned before printing results, hiding ids of files that did upload.
Now every file is still attempted, results print for each batch, and
the command fails at the end if any upload failed.
2026-10-21 15:20:40 -07:00
Devin a1d2ebbb9b operation: warm lazy log fds in the file-part leak test 2026-10-21 15:15:01 -07:00
Devin 371d320854 operation: close open file parts when a batch fails to build
Batched -dir uploads now open up to 32 files at once; a mid-batch error
left every earlier handle open until process exit. Close the parts
already built, and close the handle when Stat fails after Open.
2026-10-21 15:14:12 -07:00
Chris LuandDevin 84cc179b80 upload -dir: fetch file ids in batches instead of per file
Directory upload submitted every file through its own SubmitFiles
call, so each file paid a separate serial assign RPC. Files are now
collected during the walk and submitted in batches, sharing one batch
assign and the parallel uploads.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
Chris LuandDevin c8002d63bb upload: submit batched files concurrently
SubmitFiles fetched all file ids in one assign RPC but then uploaded
the files one after another, so a multi-file upload spent its time in
serial per-file round trips. Uploads now run through a small worker
pool while results keep their input order.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
3 changed files with 109 additions and 26 deletions

No files matched your search

+33 -19
View File
@@ -20,6 +20,10 @@ var (
upload UploadOptions
)
// uploadBatchSize is how many files a -dir upload submits per batch: one
// assign RPC fetches all their file ids, and the batch uploads in parallel.
const uploadBatchSize = 32
type UploadOptions struct {
master *string
dir *string
@@ -85,6 +89,7 @@ func runUpload(cmd *Command, args []string) bool {
if *upload.dir == "" {
return false
}
var paths []string
err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
if err == nil {
if !info.IsDir() {
@@ -93,23 +98,7 @@ func runUpload(cmd *Command, args []string) bool {
return nil
}
}
parts, e := operation.NewFileParts([]string{path})
if e != nil {
return e
}
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
Replication: *upload.replication,
Collection: *upload.collection,
DataCenter: *upload.dataCenter,
Ttl: *upload.ttl,
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
return e
}
paths = append(paths, path)
}
} else {
fmt.Println(err)
@@ -120,6 +109,31 @@ func runUpload(cmd *Command, args []string) bool {
fmt.Println(err.Error())
return false
}
hadErr := false
for batchStart := 0; batchStart < len(paths); batchStart += uploadBatchSize {
batchEnd := min(batchStart+uploadBatchSize, len(paths))
parts, e := operation.NewFileParts(paths[batchStart:batchEnd])
if e != nil {
fmt.Println(e.Error())
hadErr = true
continue
}
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
Replication: *upload.replication,
Collection: *upload.collection,
DataCenter: *upload.dataCenter,
Ttl: *upload.ttl,
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
fmt.Println(e.Error())
hadErr = true
}
}
return !hadErr
} else {
parts, e := operation.NewFileParts(args)
if e != nil {
@@ -134,12 +148,12 @@ func runUpload(cmd *Command, args []string) bool {
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if err != nil {
fmt.Println(err.Error())
return false
}
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
}
return true
}
+39 -7
View File
@@ -10,6 +10,7 @@ import (
"path"
"strconv"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -50,6 +51,10 @@ type StoragePreference struct {
type GetMasterFn func(ctx context.Context) pb.ServerAddress
// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
// once; the batch assign above already fetched all their file ids.
const submitUploadWorkers = 4
func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
results := make([]SubmitResult, len(files))
var totalBytes int64
@@ -87,21 +92,47 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
file.Server = ret.PublicUrl
}
file.Pref = pref
results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
if err != nil {
results[index].Error = err.Error()
}
results[index].Fid = file.Fid
results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
}
return results, nil
var wg sync.WaitGroup
var lastErrLock sync.Mutex
jobs := make(chan int)
for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
results[index].Size = size
if uploadErr != nil {
results[index].Error = uploadErr.Error()
lastErrLock.Lock()
err = uploadErr
lastErrLock.Unlock()
}
}
}()
}
for index := range files {
jobs <- index
}
close(jobs)
wg.Wait()
return results, err
}
func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
ret = make([]*FilePart, len(fullPathFilenames))
for index, file := range fullPathFilenames {
if ret[index], err = newFilePart(file); err != nil {
return
for _, part := range ret[:index] {
if closer, ok := part.Reader.(io.Closer); ok {
closer.Close()
}
}
return nil, err
}
}
return
@@ -113,13 +144,14 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
glog.V(0).Info("Failed to open file: ", fullPathFilename)
return ret, openErr
}
ret.Reader = fh
fi, fiErr := fh.Stat()
if fiErr != nil {
fh.Close()
glog.V(0).Info("Failed to stat file:", fullPathFilename)
return ret, fiErr
}
ret.Reader = fh
ret.ModTime = fi.ModTime().UTC().Unix()
ret.FileSize = fi.Size()
ext := strings.ToLower(path.Ext(fullPathFilename))
+37
View File
@@ -0,0 +1,37 @@
package operation
import (
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func TestNewFilePartsClosesOnError(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.txt")
if _, err := NewFileParts([]string{missing}); err == nil {
t.Skip("missing file unexpectedly opened")
}
glog.Flush()
fdsBefore, err := os.ReadDir("/proc/self/fd")
if err != nil {
t.Skip("fd counting requires /proc")
}
good := filepath.Join(dir, "good.txt")
if err := os.WriteFile(good, []byte("x"), 0644); err != nil {
t.Fatal(err)
}
if _, err := NewFileParts([]string{good, missing}); err == nil {
t.Fatal("expected error for missing file")
}
fdsAfter, err := os.ReadDir("/proc/self/fd")
if err != nil {
t.Skip("fd counting requires /proc")
}
if len(fdsAfter) != len(fdsBefore) {
t.Errorf("fd count changed by %d: a file part leaked its handle", len(fdsAfter)-len(fdsBefore))
}
}