mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-25 22:40:44 +01:00
Chris Lu
4fb3e22a01
* tiering: stop a shared remote object being deleted while replicas still point at it A remote-tiered volume's .dat content lives only in one cloud object that all N replica .vif files point at. Deleting that object while destroying any one replica, or before a downloaded replica is durable, bricks the survivors. - volume.tier.move cleanup now deletes old replicas with keepRemoteData=true so surviving replicas keep the shared object. Document why the alreadyPlaced anchor needs no replica sync (same-object replicas are byte-identical). - VolumeTierMoveDatFromRemote now fsyncs the downloaded .dat, fsyncs the containing directory, trims the .vif (fsynced) and swaps to the local DiskFile BEFORE deleting the remote object, on both the keep-remote and delete paths. Only the final DeleteFile is gated by keep_remote_dat_file, so a keep-remote download leaves the replica served from local disk rather than the shared object, and a crash before delete merely leaks the object. - volume.tier.download keeps the shared object for every replica except the last, which deletes it. - s3 and rclone download paths fsync the .dat before close. * storage: swap the volume data backend under the data lock The tier-download swap closed v.DataBackend and assigned the new local DiskFile without holding dataFileAccessLock, racing concurrent reads/writes (use of a closed file / nil deref). Add an exported Volume.SwapDataBackend that performs the close-and-replace under the lock, and call it from the tier download. * server: skip directory fsync on Windows in the tier download path os.Open(dir).Sync() is unsupported on Windows and returns an error, which would fail VolumeTierMoveDatFromRemote entirely there. Skip the directory fsync on Windows, matching how the storage-side helper tolerates the unsupported case. * shell: make multi-replica tier.download resilient to already-local replicas If a multi-replica download is interrupted and retried, a replica made local in the prior attempt returns "already on local disk", which aborted the whole command and left the remaining remote replicas dangling. Treat that case as a skip-and-continue so a retry completes the rest. * server: assert downloaded .dat content, not just length, in the tier test A length-only check passes even if the bytes are corrupted; compare the full content of the local .dat against the original.
105 lines
2.7 KiB
Go
105 lines
2.7 KiB
Go
package s3_backend
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"sync/atomic"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
)
|
|
|
|
func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string, sourceKey string,
|
|
fn func(progressed int64, percentage float32) error) (fileSize int64, err error) {
|
|
|
|
fileSize, err = getFileSize(sess, sourceBucket, sourceKey)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
//open the file
|
|
f, err := os.OpenFile(destFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to open file %q, %v", destFileName, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
// Create a downloader with the session and custom options
|
|
downloader := s3manager.NewDownloaderWithClient(sess, func(u *s3manager.Downloader) {
|
|
u.PartSize = int64(64 * 1024 * 1024)
|
|
u.Concurrency = 5
|
|
})
|
|
|
|
fileWriter := &s3DownloadProgressedWriter{
|
|
fp: f,
|
|
size: fileSize,
|
|
written: 0,
|
|
fn: fn,
|
|
}
|
|
|
|
// Download the file from S3.
|
|
fileSize, err = downloader.Download(fileWriter, &s3.GetObjectInput{
|
|
Bucket: aws.String(sourceBucket),
|
|
Key: aws.String(sourceKey),
|
|
})
|
|
if err != nil {
|
|
return fileSize, fmt.Errorf("failed to download /buckets/%s/%s to %s: %v", sourceBucket, sourceKey, destFileName, err)
|
|
}
|
|
|
|
// fsync the downloaded .dat so its content is durable before the caller trims the
|
|
// remote reference and deletes the shared remote object.
|
|
if syncErr := f.Sync(); syncErr != nil {
|
|
return fileSize, fmt.Errorf("failed to fsync %s: %v", destFileName, syncErr)
|
|
}
|
|
|
|
glog.V(1).Infof("downloaded file %s\n", destFileName)
|
|
|
|
return
|
|
}
|
|
|
|
// adapted from https://github.com/aws/aws-sdk-go/pull/1868
|
|
// and https://petersouter.xyz/s3-download-progress-bar-in-golang/
|
|
type s3DownloadProgressedWriter struct {
|
|
size int64
|
|
written int64
|
|
fn func(progressed int64, percentage float32) error
|
|
fp *os.File
|
|
}
|
|
|
|
func (w *s3DownloadProgressedWriter) WriteAt(p []byte, off int64) (int, error) {
|
|
n, err := w.fp.WriteAt(p, off)
|
|
if err != nil {
|
|
return n, err
|
|
}
|
|
|
|
// Got the length have read( or means has uploaded), and you can construct your message
|
|
atomic.AddInt64(&w.written, int64(n))
|
|
|
|
if w.fn != nil {
|
|
written := w.written
|
|
if err := w.fn(written, float32(written*100)/float32(w.size)); err != nil {
|
|
return n, err
|
|
}
|
|
}
|
|
|
|
return n, err
|
|
}
|
|
|
|
func getFileSize(svc s3iface.S3API, bucket string, key string) (filesize int64, error error) {
|
|
params := &s3.HeadObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
}
|
|
|
|
resp, err := svc.HeadObject(params)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return *resp.ContentLength, nil
|
|
}
|