mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-21 13:34:26 +02:00
Chris Lu
542495f1f1
s3: fold large chunk lists into manifest chunks on the direct write path The S3 gateway uploads chunks itself and hands the filer a fully prepared entry. On the routed write path (ObjectTransaction) the filer stores that entry as-is, so a large PutObject or CompleteMultipartUpload persisted its whole flat chunk list - a 900GB object carries 120k chunk references in one entry. Manifestize on the gateway before the entry is written, the same way mount, WebDAV, and filer.copy prepare theirs. Multipart part boundaries also record byte offsets now: the stored chunk indexes stop matching the entry once the list is folded, and partNumber reads plus GetObjectAttributes prefer the offsets. Legacy index-only records still work, with bounds checks instead of a possible panic. Copy paths resolve a manifested source into data chunks before their per-chunk copy loops - copying a manifest chunk raw would store its blob as object data still pointing at the source - and a large copied list is folded again on the destination. Completion likewise resolves manifest chunks a part entry may carry (the filer folds an oversized UploadPartCopy range) before rebasing part offsets.
62 lines
2.2 KiB
Go
62 lines
2.2 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"math"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
// saveManifestChunk returns the save function MaybeManifestize uses to store
|
|
// manifest blobs, assigning volumes against the object's real filer path so
|
|
// placement follows the bucket's storage rules.
|
|
func (s3a *S3ApiServer) saveManifestChunk(filePath string, bucket string, ttlSec int32) filer.SaveDataAsChunkFunctionType {
|
|
collection := ""
|
|
if s3a.option.FilerGroup != "" {
|
|
collection = s3a.getCollectionName(bucket)
|
|
}
|
|
return func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (*filer_pb.FileChunk, error) {
|
|
return filer.SaveGatewayDataAsChunk(filer.GatewayChunkUploadRequest{
|
|
FilerClient: s3a,
|
|
Reader: reader,
|
|
FullPath: filePath,
|
|
Offset: offset,
|
|
TsNs: tsNs,
|
|
Collection: collection,
|
|
TtlSec: ttlSec,
|
|
DataCenter: s3a.option.DataCenter,
|
|
Cipher: s3a.cipher,
|
|
})
|
|
}
|
|
}
|
|
|
|
// manifestizeChunks folds a large flat chunk list into manifest chunks
|
|
// (filer.ManifestBatch data chunks per manifest). On failure the flat list is
|
|
// returned so the write can still proceed, matching the filer's soft-fail.
|
|
func (s3a *S3ApiServer) manifestizeChunks(filePath string, bucket string, ttlSec int32, chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
|
|
manifested, err := filer.MaybeManifestize(s3a.saveManifestChunk(filePath, bucket, ttlSec), chunks)
|
|
if err != nil {
|
|
glog.V(0).Infof("MaybeManifestize %s: %v", filePath, err)
|
|
return chunks
|
|
}
|
|
return manifested
|
|
}
|
|
|
|
// flattenManifestChunks resolves any manifest chunks on a copy source into the
|
|
// flat data-chunk list, so per-chunk copy logic reads real data chunks instead
|
|
// of raw manifest blobs.
|
|
func (s3a *S3ApiServer) flattenManifestChunks(ctx context.Context, entry *filer_pb.Entry) error {
|
|
if entry == nil || !filer.HasChunkManifest(entry.GetChunks()) {
|
|
return nil
|
|
}
|
|
dataChunks, _, err := filer.ResolveChunkManifest(ctx, s3a.createLookupFileIdFunction(), entry.GetChunks(), 0, math.MaxInt64)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entry.Chunks = dataChunks
|
|
return nil
|
|
}
|