mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d84ecca2f | ||
|
|
a1d2ebbb9b | ||
|
|
371d320854 | ||
|
|
84cc179b80 | ||
|
|
c8002d63bb |
No files matched your search
+33
-19
@@ -20,6 +20,10 @@ var (
|
||||
upload UploadOptions
|
||||
)
|
||||
|
||||
// uploadBatchSize is how many files a -dir upload submits per batch: one
|
||||
// assign RPC fetches all their file ids, and the batch uploads in parallel.
|
||||
const uploadBatchSize = 32
|
||||
|
||||
type UploadOptions struct {
|
||||
master *string
|
||||
dir *string
|
||||
@@ -85,6 +89,7 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
if *upload.dir == "" {
|
||||
return false
|
||||
}
|
||||
var paths []string
|
||||
err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
|
||||
if err == nil {
|
||||
if !info.IsDir() {
|
||||
@@ -93,23 +98,7 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
parts, e := operation.NewFileParts([]string{path})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
|
||||
Replication: *upload.replication,
|
||||
Collection: *upload.collection,
|
||||
DataCenter: *upload.dataCenter,
|
||||
Ttl: *upload.ttl,
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
paths = append(paths, path)
|
||||
}
|
||||
} else {
|
||||
fmt.Println(err)
|
||||
@@ -120,6 +109,31 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
fmt.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
hadErr := false
|
||||
for batchStart := 0; batchStart < len(paths); batchStart += uploadBatchSize {
|
||||
batchEnd := min(batchStart+uploadBatchSize, len(paths))
|
||||
parts, e := operation.NewFileParts(paths[batchStart:batchEnd])
|
||||
if e != nil {
|
||||
fmt.Println(e.Error())
|
||||
hadErr = true
|
||||
continue
|
||||
}
|
||||
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
|
||||
Replication: *upload.replication,
|
||||
Collection: *upload.collection,
|
||||
DataCenter: *upload.dataCenter,
|
||||
Ttl: *upload.ttl,
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if e != nil {
|
||||
fmt.Println(e.Error())
|
||||
hadErr = true
|
||||
}
|
||||
}
|
||||
return !hadErr
|
||||
} else {
|
||||
parts, e := operation.NewFileParts(args)
|
||||
if e != nil {
|
||||
@@ -134,12 +148,12 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
@@ -50,6 +51,10 @@ type StoragePreference struct {
|
||||
|
||||
type GetMasterFn func(ctx context.Context) pb.ServerAddress
|
||||
|
||||
// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
|
||||
// once; the batch assign above already fetched all their file ids.
|
||||
const submitUploadWorkers = 4
|
||||
|
||||
func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
|
||||
results := make([]SubmitResult, len(files))
|
||||
var totalBytes int64
|
||||
@@ -87,21 +92,47 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
|
||||
file.Server = ret.PublicUrl
|
||||
}
|
||||
file.Pref = pref
|
||||
results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
|
||||
if err != nil {
|
||||
results[index].Error = err.Error()
|
||||
}
|
||||
results[index].Fid = file.Fid
|
||||
results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
|
||||
}
|
||||
return results, nil
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var lastErrLock sync.Mutex
|
||||
jobs := make(chan int)
|
||||
for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for index := range jobs {
|
||||
size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
|
||||
results[index].Size = size
|
||||
if uploadErr != nil {
|
||||
results[index].Error = uploadErr.Error()
|
||||
lastErrLock.Lock()
|
||||
err = uploadErr
|
||||
lastErrLock.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := range files {
|
||||
jobs <- index
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
return results, err
|
||||
}
|
||||
|
||||
func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
|
||||
ret = make([]*FilePart, len(fullPathFilenames))
|
||||
for index, file := range fullPathFilenames {
|
||||
if ret[index], err = newFilePart(file); err != nil {
|
||||
return
|
||||
for _, part := range ret[:index] {
|
||||
if closer, ok := part.Reader.(io.Closer); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -113,13 +144,14 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
|
||||
glog.V(0).Info("Failed to open file: ", fullPathFilename)
|
||||
return ret, openErr
|
||||
}
|
||||
ret.Reader = fh
|
||||
|
||||
fi, fiErr := fh.Stat()
|
||||
if fiErr != nil {
|
||||
fh.Close()
|
||||
glog.V(0).Info("Failed to stat file:", fullPathFilename)
|
||||
return ret, fiErr
|
||||
}
|
||||
ret.Reader = fh
|
||||
ret.ModTime = fi.ModTime().UTC().Unix()
|
||||
ret.FileSize = fi.Size()
|
||||
ext := strings.ToLower(path.Ext(fullPathFilename))
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func TestNewFilePartsClosesOnError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.txt")
|
||||
if _, err := NewFileParts([]string{missing}); err == nil {
|
||||
t.Skip("missing file unexpectedly opened")
|
||||
}
|
||||
glog.Flush()
|
||||
|
||||
fdsBefore, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
t.Skip("fd counting requires /proc")
|
||||
}
|
||||
good := filepath.Join(dir, "good.txt")
|
||||
if err := os.WriteFile(good, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := NewFileParts([]string{good, missing}); err == nil {
|
||||
t.Fatal("expected error for missing file")
|
||||
}
|
||||
fdsAfter, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
t.Skip("fd counting requires /proc")
|
||||
}
|
||||
if len(fdsAfter) != len(fdsBefore) {
|
||||
t.Errorf("fd count changed by %d: a file part leaked its handle", len(fdsAfter)-len(fdsBefore))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user