mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
* mount: re-resolve volume locations after a failed chunk read NewChunkGroup passed nil as the ReaderCache's CacheInvalidator, so retryFetchAfterCacheInvalidation was dead code on the FUSE read path. A mount that cached a volume's locations while one server was down kept retrying that server after it died, then returned EIO, even though the master and filer both resolved the live replica. The S3 gateway already passes its filerClient; do the same for the mount. * test: FUSE integration tests for volume server failover One mount appends while a second tails, and a volume server is killed, started or restarted mid-stream against a 001-replicated cluster of three volume servers. Automates the scenario matrix reported for Docker Swarm mounts, including the large-file variant and a no-chaos control. * test: report the filer's own view when append content mismatches A mismatch between what the writer wrote and what the reader sees can come from either side's cache. Read the file back through the filer's HTTP handler as well, and let the mount verbosity be raised from the environment, so a failing run says which layer lost the data. * test: wait for the reader mount to converge before comparing A mount caches metadata for about a second, so reading the file the instant the writer's last close returned can legitimately come back short. Poll the reader until it matches or the timeout expires; content that is wrong rather than merely late never converges and still fails, now with the writer's mount and the filer's own view alongside it. * test: detect a failover cluster child that exited at startup Signal(0) succeeds for a zombie and nothing reaped these children until shutdown, so a process that died on startup looked alive until the readiness timeout expired. Reap each child as it is started and consult the result. * test: read a file the killed volume server actually holds Placement decides which two of three servers back each volume, so killing volume N and reading readfile-N could pass without the victim ever holding a replica of it. Resolve each file's volumes through the filer and the master, and pick one the victim backs, preferring a file the reader has not cached. * ci: stop persisting checkout credentials in the failover workflow The job does not use the token after cloning. Also tag the README's command block as bash and match the timeout the workflow actually uses. * test: discard the ignored errors errcheck flags in the failover harness * test: resolve manifests when mapping a file to its volumes A manifest chunk's own fid names the volume holding the manifest, not the volumes holding the data, so a large enough file would point the failover victim at the wrong server. * test: pin the stale-location recovery path with a primed reader Reading a file for the first time after a server dies proves nothing: the lookup is fresh and returns the survivor. Kill one holder and wait for the master to drop it, read a file on that volume so the reader caches the lone survivor, restart the first server, then kill the survivor. The reader's only cached location is now dead while the data is live elsewhere, which is the case the invalidator exists for: EIO without it, recovery with it. * filer: re-look-up a chunk's locations as soon as they all fail A read that fails against every location it was given is far more likely to be holding a stale list than to be hitting a cluster that is briefly slow, but the retry loops spent the whole backoff ladder, about 13 s, before the caller got a chance to invalidate and look the chunk up again. Give the loops a refresh hook and let the reader cache invalidate on the first fully failed pass, so recovery starts in milliseconds. Clients without an invalidator keep the old behavior. The filer's streaming read path has its own fetch loop and is not covered. * filer: refresh locations on the random-read path too readChunkSliceAt bypasses the chunk cacher in random-access mode and fetches the range directly, which left it without the invalidation the cacher does: a random reader parked on a stale location had no way back at all. Hoist the refresh hook onto the reader cache so both paths share it. * filer: compare chunk locations as a set, not in order Lookups shuffle the locations they return, so comparing positionally reads a reshuffle of the very same replicas as a fresh set and spends an immediate retry on locations that just failed. weed/filer already had an order-independent comparison for this; move it next to the retry loops so both callers share one helper.
285 lines
8.4 KiB
Go
285 lines
8.4 KiB
Go
package filer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
)
|
|
|
|
const (
|
|
ManifestBatch = 10000
|
|
)
|
|
|
|
var bytesBufferPool = sync.Pool{
|
|
New: func() interface{} {
|
|
return new(bytes.Buffer)
|
|
},
|
|
}
|
|
|
|
func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
|
|
for _, chunk := range chunks {
|
|
if chunk.IsChunkManifest {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonManifestChunks []*filer_pb.FileChunk) {
|
|
for _, c := range chunks {
|
|
if c.IsChunkManifest {
|
|
manifestChunks = append(manifestChunks, c)
|
|
} else {
|
|
nonManifestChunks = append(nonManifestChunks, c)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
|
|
// TODO maybe parallel this
|
|
for _, chunk := range chunks {
|
|
|
|
if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
|
|
continue
|
|
}
|
|
|
|
if !chunk.IsChunkManifest {
|
|
dataChunks = append(dataChunks, chunk)
|
|
continue
|
|
}
|
|
|
|
resolvedChunks, err := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk)
|
|
if err != nil {
|
|
return dataChunks, nil, err
|
|
}
|
|
|
|
manifestChunks = append(manifestChunks, chunk)
|
|
// recursive
|
|
subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
|
|
if subErr != nil {
|
|
return dataChunks, nil, subErr
|
|
}
|
|
dataChunks = append(dataChunks, subDataChunks...)
|
|
manifestChunks = append(manifestChunks, subManifestChunks...)
|
|
}
|
|
return
|
|
}
|
|
|
|
func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
|
|
if !chunk.IsChunkManifest {
|
|
return
|
|
}
|
|
|
|
// IsChunkManifest
|
|
bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
|
|
bytesBuffer.Reset()
|
|
defer bytesBufferPool.Put(bytesBuffer)
|
|
err := fetchWholeChunk(ctx, bytesBuffer, lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)
|
|
}
|
|
m := &filer_pb.FileChunkManifest{}
|
|
if err := proto.Unmarshal(bytesBuffer.Bytes(), m); err != nil {
|
|
return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err)
|
|
}
|
|
|
|
// recursive
|
|
filer_pb.AfterEntryDeserialization(m.Chunks)
|
|
return m.Chunks, nil
|
|
}
|
|
|
|
// TODO fetch from cache for weed mount?
|
|
func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) error {
|
|
urlStrings, err := lookupFileIdFn(ctx, fileId)
|
|
if err != nil {
|
|
glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
|
|
return err
|
|
}
|
|
jwt := JwtForVolumeServer(fileId)
|
|
_, err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, jwt, cipherKey, isGzipped, true, 0, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fetchChunkRange(ctx context.Context, buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, refreshUrls util_http.RefreshUrlsFunc) (int, error) {
|
|
urlStrings, err := lookupFileIdFn(ctx, fileId)
|
|
if err != nil {
|
|
glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
|
|
return 0, err
|
|
}
|
|
return util_http.RetriedFetchChunkData(ctx, buffer, urlStrings, cipherKey, isGzipped, false, offset, fileId, refreshUrls)
|
|
}
|
|
|
|
func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (written int64, err error) {
|
|
|
|
var shouldRetry bool
|
|
var totalWritten int
|
|
|
|
for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
|
|
// Check for context cancellation before starting retry loop
|
|
select {
|
|
case <-ctx.Done():
|
|
return int64(totalWritten), ctx.Err()
|
|
default:
|
|
}
|
|
|
|
retriedCnt := 0
|
|
for _, urlString := range urlStrings {
|
|
// Check for context cancellation before each volume server request
|
|
select {
|
|
case <-ctx.Done():
|
|
return int64(totalWritten), ctx.Err()
|
|
default:
|
|
}
|
|
|
|
retriedCnt++
|
|
var localProcessed int
|
|
var writeErr error
|
|
shouldRetry, err = util_http.ReadUrlAsStream(ctx, util_http.AppendQueryParameter(urlString, "readDeleted", "true"), jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
|
|
// Check for context cancellation during data processing
|
|
select {
|
|
case <-ctx.Done():
|
|
writeErr = ctx.Err()
|
|
return
|
|
default:
|
|
}
|
|
|
|
if totalWritten > localProcessed {
|
|
toBeSkipped := totalWritten - localProcessed
|
|
if len(data) <= toBeSkipped {
|
|
localProcessed += len(data)
|
|
return // skip if already processed
|
|
}
|
|
data = data[toBeSkipped:]
|
|
localProcessed += toBeSkipped
|
|
}
|
|
var writtenCount int
|
|
writtenCount, writeErr = writer.Write(data)
|
|
localProcessed += writtenCount
|
|
totalWritten += writtenCount
|
|
})
|
|
if !shouldRetry {
|
|
break
|
|
}
|
|
if writeErr != nil {
|
|
err = writeErr
|
|
break
|
|
}
|
|
if err != nil {
|
|
glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
// all nodes have tried it
|
|
if retriedCnt == len(urlStrings) {
|
|
break
|
|
}
|
|
if err != nil && shouldRetry {
|
|
glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
|
|
// Sleep with proper context cancellation and timer cleanup
|
|
timer := time.NewTimer(waitTime)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return int64(totalWritten), ctx.Err()
|
|
case <-timer.C:
|
|
// Continue with retry
|
|
}
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
|
|
return int64(totalWritten), err
|
|
|
|
}
|
|
|
|
func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
|
|
// Don't manifestize SSE-encrypted chunks to preserve per-chunk metadata
|
|
for _, chunk := range inputChunks {
|
|
if chunk.GetSseType() != 0 { // Any SSE type (SSE-C or SSE-KMS)
|
|
return inputChunks, nil
|
|
}
|
|
}
|
|
return doMaybeManifestize(saveFunc, inputChunks, ManifestBatch, mergeIntoManifest)
|
|
}
|
|
|
|
func doMaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk, mergeFactor int, mergefn func(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error)) (chunks []*filer_pb.FileChunk, err error) {
|
|
|
|
var dataChunks []*filer_pb.FileChunk
|
|
for _, chunk := range inputChunks {
|
|
if !chunk.IsChunkManifest {
|
|
dataChunks = append(dataChunks, chunk)
|
|
} else {
|
|
chunks = append(chunks, chunk)
|
|
}
|
|
}
|
|
|
|
remaining := len(dataChunks)
|
|
for i := 0; i+mergeFactor <= len(dataChunks); i += mergeFactor {
|
|
chunk, err := mergefn(saveFunc, dataChunks[i:i+mergeFactor])
|
|
if err != nil {
|
|
return dataChunks, err
|
|
}
|
|
chunks = append(chunks, chunk)
|
|
remaining -= mergeFactor
|
|
}
|
|
// remaining
|
|
for i := len(dataChunks) - remaining; i < len(dataChunks); i++ {
|
|
chunks = append(chunks, dataChunks[i])
|
|
}
|
|
return
|
|
}
|
|
|
|
func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {
|
|
|
|
filer_pb.BeforeEntrySerialization(dataChunks)
|
|
|
|
// create and serialize the manifest
|
|
data, serErr := proto.Marshal(&filer_pb.FileChunkManifest{
|
|
Chunks: dataChunks,
|
|
})
|
|
if serErr != nil {
|
|
return nil, fmt.Errorf("serializing manifest: %w", serErr)
|
|
}
|
|
|
|
minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
|
|
for _, chunk := range dataChunks {
|
|
if minOffset > int64(chunk.Offset) {
|
|
minOffset = chunk.Offset
|
|
}
|
|
if maxOffset < int64(chunk.Size)+chunk.Offset {
|
|
maxOffset = int64(chunk.Size) + chunk.Offset
|
|
}
|
|
}
|
|
|
|
manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0, 0, uint64(len(data)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
manifestChunk.IsChunkManifest = true
|
|
manifestChunk.Offset = minOffset
|
|
manifestChunk.Size = uint64(maxOffset - minOffset)
|
|
|
|
return
|
|
}
|
|
|
|
type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (chunk *filer_pb.FileChunk, err error)
|