mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* volume: fsync a batch delete that asks for it BatchDeleteRequest gets an fsync field, and the Go and Rust volume servers pass it to the needle delete, which already orders the tombstone, .dat sync, publish and .idx sync for a durable delete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * operation: pass an fsync flag to batch deletes DeleteFileIdsWithLookupVolumeId and DeleteFileIdsAtOneVolumeServer take an fsync argument and send it in the BatchDeleteRequest. Every caller passes false for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: fsync chunk deletes under an fsync path rule DeleteChunks passes the path rule's fsync flag to the deletion queue. The flag is kept with the pending record until the delete is confirmed, so retries and re-queues keep it, and the batch and retry loops send marked ids in a durable BatchDelete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep fsync marks in the deletion ledger Marked ids go into the trailing parts of a manifest ledger, counted by fsyncParts. Older filers read those parts as plain ids and still delete them; a reload or a claimed foreign ledger restores the marks. A ledger without marks keeps the single array form. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: record a queued deletion before the delete loop can see it queueDeletions published the ids to the hot queue before taking the ledger lock, so a concurrent flush could delete an id before its fsync mark landed and then forget the mark with the record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: fsync a folder delete's chunks by each child's path rule A recursive delete collected its file children's chunks and resolved the rule once for the folder, which misses a trailing-slash rule on the folder itself and file-prefix rules. Each file child's own rule now decides fsync; the folder rule still decides chunk deletion. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a newer fsync request when a plain delete finishes An id re-queued with fsync while a plain delete of it was in flight lost its mark when that delete succeeded, so the queued copy also ran without fsync. A plain attempt now leaves a marked record for its queued copy; a durable delete of an already-deleted needle still syncs the tombstone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: delete a folder's plain child chunks without fsync Children whose own rule turns fsync off were still deleted durably when the folder's rule asked for fsync. Only the children marked by their own rule now go out with fsync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: retry a durable chunk delete the volume server failed A volume server rolls back a durable delete it cannot sync and answers with a 5xx carrying the I/O error text, which the classifier took as permanent, so the record and its fsync mark were dropped while the needle stayed live. A 5xx answer to a durable delete is now retryable, within the usual attempt limit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
705 lines
25 KiB
Go
705 lines
25 KiB
Go
package filer
|
|
|
|
import (
|
|
"container/heap"
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"google.golang.org/grpc"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
|
)
|
|
|
|
const (
|
|
// Maximum number of retry attempts for failed deletions
|
|
MaxRetryAttempts = 10
|
|
// Initial retry delay (will be doubled with each attempt)
|
|
InitialRetryDelay = 5 * time.Minute
|
|
// Maximum retry delay
|
|
MaxRetryDelay = 6 * time.Hour
|
|
// Interval for checking retry queue for ready items
|
|
DeletionRetryPollInterval = 1 * time.Minute
|
|
// Maximum number of items to process per retry iteration
|
|
DeletionRetryBatchSize = 1000
|
|
// Maximum number of error details to include in log messages
|
|
MaxLoggedErrorDetails = 10
|
|
// Interval for polling the deletion queue for new items
|
|
// Using a prime number to de-synchronize with other periodic tasks
|
|
DeletionPollInterval = 1123 * time.Millisecond
|
|
// Maximum number of file IDs to delete per batch (roughly 20 bytes per file ID)
|
|
DeletionBatchSize = 100000
|
|
)
|
|
|
|
// retryablePatterns contains transient conditions specific to the deletion
|
|
// pipeline, on top of the network and service failures util.IsTransientErrorMessage
|
|
// already covers.
|
|
//
|
|
// Context cancellation counts as retryable here but not in util: this decides
|
|
// whether to requeue the deletion, not whether to retry a call, and a cancelled
|
|
// batch leaves the file still needing deletion.
|
|
var retryablePatterns = []string{
|
|
"is read only", // Volume temporarily read-only (tiering, maintenance)
|
|
"error reading from server", // Network I/O errors
|
|
"closed network connection", // Network connection closed unexpectedly
|
|
"timeout", // Operation timeout (network or server)
|
|
"deadline exceeded", // Context deadline exceeded
|
|
"context canceled", // Context cancellation
|
|
"lookup error", // Volume lookup failures
|
|
"lookup failed", // Volume server discovery issues
|
|
"too many requests", // Rate limiting / backpressure
|
|
"try again", // Explicit retry suggestion
|
|
}
|
|
|
|
// DeletionRetryItem represents a file deletion that failed and needs to be retried
|
|
type DeletionRetryItem struct {
|
|
FileId string
|
|
RetryCount int
|
|
NextRetryAt time.Time
|
|
LastError string
|
|
ledgerEpoch uint64 // pendingDeletions epoch when the item was queued; expiry forgets only a matching epoch
|
|
heapIndex int // index in the heap (for heap.Interface)
|
|
inFlight bool // true when item is being processed, prevents duplicate additions
|
|
}
|
|
|
|
// retryHeap implements heap.Interface for DeletionRetryItem
|
|
// Items are ordered by NextRetryAt (earliest first)
|
|
type retryHeap []*DeletionRetryItem
|
|
|
|
// Compile-time assertion that retryHeap implements heap.Interface
|
|
var _ heap.Interface = (*retryHeap)(nil)
|
|
|
|
func (h retryHeap) Len() int { return len(h) }
|
|
|
|
func (h retryHeap) Less(i, j int) bool {
|
|
return h[i].NextRetryAt.Before(h[j].NextRetryAt)
|
|
}
|
|
|
|
func (h retryHeap) Swap(i, j int) {
|
|
h[i], h[j] = h[j], h[i]
|
|
h[i].heapIndex = i
|
|
h[j].heapIndex = j
|
|
}
|
|
|
|
func (h *retryHeap) Push(x any) {
|
|
item := x.(*DeletionRetryItem)
|
|
item.heapIndex = len(*h)
|
|
*h = append(*h, item)
|
|
}
|
|
|
|
func (h *retryHeap) Pop() any {
|
|
old := *h
|
|
n := len(old)
|
|
item := old[n-1]
|
|
old[n-1] = nil // avoid memory leak
|
|
item.heapIndex = -1 // mark as removed
|
|
*h = old[0 : n-1]
|
|
return item
|
|
}
|
|
|
|
// DeletionRetryQueue manages the queue of failed deletions that need to be retried.
|
|
// Uses a min-heap ordered by NextRetryAt for efficient retrieval of ready items.
|
|
//
|
|
// LIMITATION: Current implementation stores retry queue in memory only.
|
|
// On filer restart, all pending retries are lost. With MaxRetryDelay up to 6 hours,
|
|
// process restarts during this window will cause retry state loss.
|
|
//
|
|
// TODO: Consider persisting retry queue to durable storage for production resilience:
|
|
// - Option 1: Leverage existing Filer store (KV operations)
|
|
// - Option 2: Periodic snapshots to disk with recovery on startup
|
|
// - Option 3: Write-ahead log for retry queue mutations
|
|
// - Trade-offs: Performance vs durability, complexity vs reliability
|
|
//
|
|
// For now, accepting in-memory storage as pragmatic initial implementation.
|
|
// Lost retries will be eventually consistent as files remain in deletion queue.
|
|
type DeletionRetryQueue struct {
|
|
heap retryHeap
|
|
itemIndex map[string]*DeletionRetryItem // for O(1) lookup by FileId
|
|
lock sync.Mutex
|
|
}
|
|
|
|
// NewDeletionRetryQueue creates a new retry queue
|
|
func NewDeletionRetryQueue() *DeletionRetryQueue {
|
|
q := &DeletionRetryQueue{
|
|
heap: make(retryHeap, 0),
|
|
itemIndex: make(map[string]*DeletionRetryItem),
|
|
}
|
|
heap.Init(&q.heap)
|
|
return q
|
|
}
|
|
|
|
// calculateBackoff calculates the exponential backoff delay for a given retry count.
|
|
// Uses exponential backoff formula: InitialRetryDelay * 2^(retryCount-1)
|
|
// The first retry (retryCount=1) uses InitialRetryDelay, second uses 2x, third uses 4x, etc.
|
|
// Includes overflow protection and caps at MaxRetryDelay.
|
|
func calculateBackoff(retryCount int) time.Duration {
|
|
// The first retry is attempt 1, but shift should start at 0
|
|
if retryCount <= 1 {
|
|
return InitialRetryDelay
|
|
}
|
|
|
|
shiftAmount := uint(retryCount - 1)
|
|
|
|
// time.Duration is an int64. A left shift of 63 or more will result in a
|
|
// negative number or zero. The multiplication can also overflow much earlier
|
|
// (around a shift of 25 for a 5-minute initial delay).
|
|
// The `delay <= 0` check below correctly catches all these overflow cases.
|
|
delay := InitialRetryDelay << shiftAmount
|
|
|
|
if delay <= 0 || delay > MaxRetryDelay {
|
|
return MaxRetryDelay
|
|
}
|
|
|
|
return delay
|
|
}
|
|
|
|
// AddOrUpdate adds a new failed deletion or updates an existing one
|
|
// Time complexity: O(log N) for insertion/update
|
|
func (q *DeletionRetryQueue) AddOrUpdate(fileId string, errorMsg string, ledgerEpoch uint64) {
|
|
q.lock.Lock()
|
|
defer q.lock.Unlock()
|
|
|
|
// Check if item already exists (including in-flight items)
|
|
if item, exists := q.itemIndex[fileId]; exists {
|
|
// Item is already in the queue or being processed. Just update the error.
|
|
// The existing retry schedule should proceed.
|
|
// RetryCount is only incremented in RequeueForRetry when an actual retry is performed.
|
|
item.LastError = errorMsg
|
|
// Keep the recorded epoch while the item is in flight: an expiry or
|
|
// permanent outcome from that attempt must forget only the record it
|
|
// started with, not a newer enqueue for the same id. This also keeps
|
|
// the field immutable once the worker can read it without the lock.
|
|
if !item.inFlight {
|
|
item.ledgerEpoch = ledgerEpoch
|
|
}
|
|
if item.inFlight {
|
|
glog.V(2).Infof("retry for %s in-flight: attempt %d, will preserve retry state", fileId, item.RetryCount)
|
|
} else {
|
|
glog.V(2).Infof("retry for %s already scheduled: attempt %d, next retry in %v", fileId, item.RetryCount, time.Until(item.NextRetryAt))
|
|
}
|
|
return
|
|
}
|
|
|
|
// Add new item
|
|
delay := InitialRetryDelay
|
|
item := &DeletionRetryItem{
|
|
FileId: fileId,
|
|
RetryCount: 1,
|
|
NextRetryAt: time.Now().Add(delay),
|
|
LastError: errorMsg,
|
|
ledgerEpoch: ledgerEpoch,
|
|
inFlight: false,
|
|
}
|
|
heap.Push(&q.heap, item)
|
|
q.itemIndex[fileId] = item
|
|
glog.V(2).Infof("added retry for %s: next retry in %v", fileId, delay)
|
|
}
|
|
|
|
// RequeueForRetry re-adds a previously failed item back to the queue with incremented retry count.
|
|
// This method MUST be used when re-queuing items from processRetryBatch to preserve retry state.
|
|
// Time complexity: O(log N) for insertion
|
|
func (q *DeletionRetryQueue) RequeueForRetry(item *DeletionRetryItem, errorMsg string) {
|
|
q.lock.Lock()
|
|
defer q.lock.Unlock()
|
|
|
|
// Increment retry count
|
|
item.RetryCount++
|
|
item.LastError = errorMsg
|
|
|
|
// Calculate next retry time with exponential backoff
|
|
delay := calculateBackoff(item.RetryCount)
|
|
item.NextRetryAt = time.Now().Add(delay)
|
|
item.inFlight = false // Clear in-flight flag
|
|
glog.V(2).Infof("requeued retry for %s: attempt %d, next retry in %v", item.FileId, item.RetryCount, delay)
|
|
|
|
// Re-add to heap (item still in itemIndex)
|
|
heap.Push(&q.heap, item)
|
|
}
|
|
|
|
// GetReadyItems returns items that are ready to be retried and marks them as in-flight.
|
|
// Expired reports fileIds dropped for exceeding MaxRetryAttempts — permanent
|
|
// dropouts the caller must also forget in the durable deletion ledger.
|
|
// Time complexity: O(K log N) where K is the number of ready items
|
|
// Items are processed in order of NextRetryAt (earliest first)
|
|
func (q *DeletionRetryQueue) GetReadyItems(maxItems int) (ready []*DeletionRetryItem, expired []*DeletionRetryItem) {
|
|
q.lock.Lock()
|
|
defer q.lock.Unlock()
|
|
|
|
now := time.Now()
|
|
|
|
// Peek at items from the top of the heap (earliest NextRetryAt)
|
|
for len(q.heap) > 0 && len(ready) < maxItems {
|
|
item := q.heap[0]
|
|
|
|
// If the earliest item is not ready yet, no other items are ready either
|
|
if item.NextRetryAt.After(now) {
|
|
break
|
|
}
|
|
|
|
// Remove from heap but keep in itemIndex with inFlight flag
|
|
heap.Pop(&q.heap)
|
|
|
|
if item.RetryCount <= MaxRetryAttempts {
|
|
item.inFlight = true // Mark as being processed
|
|
ready = append(ready, item)
|
|
} else {
|
|
// Max attempts reached, log and discard completely
|
|
delete(q.itemIndex, item.FileId)
|
|
expired = append(expired, item)
|
|
glog.Warningf("max retry attempts (%d) reached for %s, last error: %s", MaxRetryAttempts, item.FileId, item.LastError)
|
|
}
|
|
}
|
|
|
|
return ready, expired
|
|
}
|
|
|
|
// Remove removes an item from the queue (called when deletion succeeds or fails permanently)
|
|
// Time complexity: O(1)
|
|
func (q *DeletionRetryQueue) Remove(item *DeletionRetryItem) {
|
|
q.lock.Lock()
|
|
defer q.lock.Unlock()
|
|
|
|
// Item was already removed from heap by GetReadyItems, just remove from index
|
|
delete(q.itemIndex, item.FileId)
|
|
}
|
|
|
|
// Size returns the current size of the retry queue
|
|
func (q *DeletionRetryQueue) Size() int {
|
|
q.lock.Lock()
|
|
defer q.lock.Unlock()
|
|
return len(q.heap)
|
|
}
|
|
|
|
func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]*operation.LookupResult, error) {
|
|
return func(vids []string) (map[string]*operation.LookupResult, error) {
|
|
m := make(map[string]*operation.LookupResult)
|
|
for _, vid := range vids {
|
|
locs, _ := masterClient.GetVidLocations(vid)
|
|
var locations []operation.Location
|
|
for _, loc := range locs {
|
|
locations = append(locations, operation.Location{
|
|
Url: loc.Url,
|
|
PublicUrl: loc.PublicUrl,
|
|
GrpcPort: loc.GrpcPort,
|
|
})
|
|
}
|
|
m[vid] = &operation.LookupResult{
|
|
VolumeOrFileId: vid,
|
|
Locations: locations,
|
|
}
|
|
}
|
|
return m, nil
|
|
}
|
|
}
|
|
|
|
func (f *Filer) loopProcessingDeletion() {
|
|
|
|
lookupFunc := LookupByMasterClientFn(f.MasterClient)
|
|
|
|
// Start retry processor in a separate goroutine
|
|
go f.loopProcessingDeletionRetry(lookupFunc)
|
|
|
|
ticker := time.NewTicker(DeletionPollInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-f.deletionQuit:
|
|
glog.V(0).Infof("deletion processor shutting down")
|
|
return
|
|
case <-ticker.C:
|
|
f.FlushFileIdDeletionQueue(context.Background(), lookupFunc)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *Filer) FlushFileIdDeletionQueue(ctx context.Context, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) (consumed []string) {
|
|
f.FileIdDeletionQueue.Consume(func(fileIds []string) {
|
|
consumed = fileIds
|
|
for i := 0; i < len(fileIds); i += DeletionBatchSize {
|
|
end := i + DeletionBatchSize
|
|
if end > len(fileIds) {
|
|
end = len(fileIds)
|
|
}
|
|
f.processDeletionBatch(ctx, fileIds[i:end], lookupFunc)
|
|
}
|
|
})
|
|
return consumed
|
|
}
|
|
|
|
// processDeletionBatch handles deletion of a batch of file IDs and processes results.
|
|
// It classifies errors into retryable and permanent categories, adds retryable failures
|
|
// to the retry queue, and logs appropriate messages.
|
|
func (f *Filer) processDeletionBatch(ctx context.Context, toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
|
|
// Deduplicate file IDs to prevent incorrect retry count increments for the same file ID within a single batch.
|
|
uniqueFileIdsSlice := make([]string, 0, len(toDeleteFileIds))
|
|
processed := make(map[string]struct{}, len(toDeleteFileIds))
|
|
for _, fileId := range toDeleteFileIds {
|
|
if _, found := processed[fileId]; !found {
|
|
processed[fileId] = struct{}{}
|
|
uniqueFileIdsSlice = append(uniqueFileIdsSlice, fileId)
|
|
}
|
|
}
|
|
|
|
if len(uniqueFileIdsSlice) == 0 {
|
|
return
|
|
}
|
|
|
|
// Remember each id's ledger epoch before the remote deletes run: a
|
|
// permanent outcome below must not erase a record re-queued mid-flight.
|
|
epochs := make(map[string]uint64, len(uniqueFileIdsSlice))
|
|
for _, fileId := range uniqueFileIdsSlice {
|
|
epochs[fileId] = f.deletionEpoch(fileId)
|
|
}
|
|
|
|
// Delete files and classify outcomes
|
|
fsyncIds := f.fsyncDeletionIds(uniqueFileIdsSlice)
|
|
outcomes := deleteFilesAndClassify(ctx, f.volumeGrpcDialOption(), uniqueFileIdsSlice, fsyncIds, lookupFunc)
|
|
|
|
// Process outcomes
|
|
var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int
|
|
var errorDetails []string
|
|
|
|
for _, fileId := range uniqueFileIdsSlice {
|
|
outcome := outcomes[fileId]
|
|
_, fsynced := fsyncIds[fileId]
|
|
|
|
switch outcome.status {
|
|
case deletionOutcomeSuccess:
|
|
successCount++
|
|
f.forgetDeletion(fileId, fsynced) // confirmed gone: drop from durable ledger
|
|
case deletionOutcomeNotFound:
|
|
notFoundCount++
|
|
f.forgetDeletion(fileId, fsynced) // already gone: drop from durable ledger
|
|
case deletionOutcomeRetryable, deletionOutcomeNoResult:
|
|
retryableErrorCount++
|
|
// Keep in the durable ledger: not confirmed, must survive restarts.
|
|
f.DeletionRetryQueue.AddOrUpdate(fileId, outcome.errorMsg, f.deletionEpoch(fileId))
|
|
if len(errorDetails) < MaxLoggedErrorDetails {
|
|
errorDetails = append(errorDetails, fileId+": "+outcome.errorMsg+" (will retry)")
|
|
}
|
|
case deletionOutcomePermanent:
|
|
permanentErrorCount++
|
|
// gave up: drop the record, unless the id was re-queued meanwhile
|
|
f.forgetDeletionEpoch(fileId, epochs[fileId])
|
|
if len(errorDetails) < MaxLoggedErrorDetails {
|
|
errorDetails = append(errorDetails, fileId+": "+outcome.errorMsg+" (permanent)")
|
|
}
|
|
}
|
|
}
|
|
|
|
if successCount > 0 || notFoundCount > 0 {
|
|
glog.V(2).Infof("deleted %d files successfully, %d already deleted (not found)", successCount, notFoundCount)
|
|
}
|
|
|
|
totalErrors := retryableErrorCount + permanentErrorCount
|
|
if totalErrors > 0 {
|
|
logMessage := fmt.Sprintf("failed to delete %d/%d files (%d retryable, %d permanent)",
|
|
totalErrors, len(uniqueFileIdsSlice), retryableErrorCount, permanentErrorCount)
|
|
if len(errorDetails) > 0 {
|
|
if totalErrors > MaxLoggedErrorDetails {
|
|
logMessage += fmt.Sprintf(" (showing first %d)", len(errorDetails))
|
|
}
|
|
glog.V(0).Infof("%s: %v", logMessage, strings.Join(errorDetails, "; "))
|
|
} else {
|
|
glog.V(0).Info(logMessage)
|
|
}
|
|
}
|
|
|
|
if f.DeletionRetryQueue.Size() > 0 {
|
|
glog.V(2).Infof("retry queue size: %d", f.DeletionRetryQueue.Size())
|
|
}
|
|
}
|
|
|
|
const (
|
|
deletionOutcomeSuccess = "success"
|
|
deletionOutcomeNotFound = "not_found"
|
|
deletionOutcomeRetryable = "retryable"
|
|
deletionOutcomePermanent = "permanent"
|
|
deletionOutcomeNoResult = "no_result"
|
|
)
|
|
|
|
// deletionOutcome represents the result of classifying deletion results for a file
|
|
type deletionOutcome struct {
|
|
status string // One of the deletionOutcome* constants
|
|
errorMsg string
|
|
}
|
|
|
|
// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs.
|
|
// Ids in fsyncIds are deleted durably.
|
|
func deleteFilesAndClassify(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, fsyncIds map[string]struct{}, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
|
|
var plainIds, syncedIds []string
|
|
for _, fileId := range fileIds {
|
|
if _, ok := fsyncIds[fileId]; ok {
|
|
syncedIds = append(syncedIds, fileId)
|
|
} else {
|
|
plainIds = append(plainIds, fileId)
|
|
}
|
|
}
|
|
var results []*volume_server_pb.DeleteResult
|
|
if len(plainIds) > 0 {
|
|
results = operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, plainIds, false, lookupFunc)
|
|
}
|
|
if len(syncedIds) > 0 {
|
|
results = append(results, operation.DeleteFileIdsWithLookupVolumeId(ctx, grpcDialOption, syncedIds, true, lookupFunc)...)
|
|
}
|
|
|
|
// Group results by file ID to handle multiple results for replicated volumes
|
|
resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult)
|
|
for _, result := range results {
|
|
resultsByFileId[result.FileId] = append(resultsByFileId[result.FileId], result)
|
|
}
|
|
|
|
// Classify outcome for each file
|
|
outcomes := make(map[string]deletionOutcome, len(fileIds))
|
|
for _, fileId := range fileIds {
|
|
_, durable := fsyncIds[fileId]
|
|
outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId, durable)
|
|
}
|
|
|
|
return outcomes
|
|
}
|
|
|
|
// classifyDeletionOutcome examines all deletion results for a file ID and determines the overall outcome
|
|
// Uses a single pass through results with early return for permanent errors (highest priority)
|
|
// Priority: Permanent > Retryable > Success > Not Found
|
|
// A durable delete the volume server could not sync is rolled back and
|
|
// answered with a 5xx, so for a durable delete that answer stays retryable.
|
|
func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult, durable bool) deletionOutcome {
|
|
fileIdResults, found := resultsByFileId[fileId]
|
|
if !found || len(fileIdResults) == 0 {
|
|
return deletionOutcome{
|
|
status: deletionOutcomeNoResult,
|
|
errorMsg: "no deletion result from volume server",
|
|
}
|
|
}
|
|
|
|
var firstRetryableError string
|
|
hasSuccess := false
|
|
|
|
for _, res := range fileIdResults {
|
|
if res.Error == "" {
|
|
hasSuccess = true
|
|
continue
|
|
}
|
|
if strings.Contains(res.Error, storage.ErrorDeleted.Error()) || res.Error == "not found" {
|
|
continue
|
|
}
|
|
|
|
if isRetryableError(res.Error) || (durable && res.Status >= http.StatusInternalServerError) {
|
|
if firstRetryableError == "" {
|
|
firstRetryableError = res.Error
|
|
}
|
|
} else {
|
|
// Permanent error takes highest precedence - return immediately
|
|
return deletionOutcome{status: deletionOutcomePermanent, errorMsg: res.Error}
|
|
}
|
|
}
|
|
|
|
if firstRetryableError != "" {
|
|
return deletionOutcome{status: deletionOutcomeRetryable, errorMsg: firstRetryableError}
|
|
}
|
|
|
|
if hasSuccess {
|
|
return deletionOutcome{status: deletionOutcomeSuccess, errorMsg: ""}
|
|
}
|
|
|
|
// If we are here, all results were "not found"
|
|
return deletionOutcome{status: deletionOutcomeNotFound, errorMsg: ""}
|
|
}
|
|
|
|
// isRetryableError determines if an error is retryable based on its message.
|
|
//
|
|
// Current implementation uses string matching which is brittle and may break
|
|
// if error messages change in dependencies. This is acceptable for the initial
|
|
// implementation but should be improved in the future.
|
|
//
|
|
// TODO: Consider these improvements for more robust error handling:
|
|
// - Pass DeleteResult instead of just error string to access Status codes
|
|
// - Use HTTP status codes (503 Service Unavailable, 429 Too Many Requests, etc.)
|
|
// - Implement structured error types that can be checked with errors.Is/errors.As
|
|
// - Extract and check gRPC status codes for better classification
|
|
// - Add error wrapping in the deletion pipeline to preserve error context
|
|
//
|
|
// For now, we use conservative string matching for known transient error patterns.
|
|
func isRetryableError(errorMsg string) bool {
|
|
// Empty errors are not retryable
|
|
if errorMsg == "" {
|
|
return false
|
|
}
|
|
|
|
if util.IsTransientErrorMessage(errorMsg) {
|
|
return true
|
|
}
|
|
|
|
errorLower := strings.ToLower(errorMsg)
|
|
for _, pattern := range retryablePatterns {
|
|
if strings.Contains(errorLower, pattern) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// loopProcessingDeletionRetry processes the retry queue for failed deletions
|
|
func (f *Filer) loopProcessingDeletionRetry(lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
|
|
|
|
ticker := time.NewTicker(DeletionRetryPollInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-f.deletionQuit:
|
|
glog.V(0).Infof("retry processor shutting down, %d items remaining in queue", f.DeletionRetryQueue.Size())
|
|
return
|
|
case <-ticker.C:
|
|
// Process all ready items in batches until queue is empty
|
|
totalProcessed := 0
|
|
for {
|
|
readyItems, expired := f.DeletionRetryQueue.GetReadyItems(DeletionRetryBatchSize)
|
|
for _, item := range expired {
|
|
// Permanently discarded — drop the ledger record, but only
|
|
// if the id was not re-queued after this retry item was
|
|
// recorded. A surviving newer record still needs work, so
|
|
// it goes back through the hot queue.
|
|
if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) {
|
|
f.queueDeletions(false, item.FileId)
|
|
}
|
|
}
|
|
if len(readyItems) == 0 {
|
|
break
|
|
}
|
|
|
|
f.processRetryBatch(readyItems, lookupFunc)
|
|
totalProcessed += len(readyItems)
|
|
}
|
|
|
|
if totalProcessed > 0 {
|
|
glog.V(1).Infof("retried deletion of %d files", totalProcessed)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// processRetryBatch attempts to retry deletion of files and processes results.
|
|
// Successfully deleted items are removed from tracking, retryable failures are
|
|
// re-queued with updated retry counts, and permanent errors are logged and discarded.
|
|
func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
|
|
// Extract file IDs from retry items
|
|
fileIds := make([]string, 0, len(readyItems))
|
|
for _, item := range readyItems {
|
|
fileIds = append(fileIds, item.FileId)
|
|
}
|
|
|
|
// Delete files and classify outcomes
|
|
fsyncIds := f.fsyncDeletionIds(fileIds)
|
|
outcomes := deleteFilesAndClassify(context.Background(), f.volumeGrpcDialOption(), fileIds, fsyncIds, lookupFunc)
|
|
|
|
// Process outcomes - iterate over readyItems to ensure all items are accounted for
|
|
var successCount, notFoundCount, retryCount, permanentErrorCount int
|
|
for _, item := range readyItems {
|
|
outcome := outcomes[item.FileId]
|
|
_, fsynced := fsyncIds[item.FileId]
|
|
|
|
switch outcome.status {
|
|
case deletionOutcomeSuccess:
|
|
successCount++
|
|
f.DeletionRetryQueue.Remove(item) // Remove from queue (success)
|
|
f.forgetDeletion(item.FileId, fsynced) // confirmed gone: drop from durable ledger
|
|
glog.V(2).Infof("retry successful for %s after %d attempts", item.FileId, item.RetryCount)
|
|
case deletionOutcomeNotFound:
|
|
notFoundCount++
|
|
f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted)
|
|
f.forgetDeletion(item.FileId, fsynced) // already gone: drop from durable ledger
|
|
case deletionOutcomeRetryable, deletionOutcomeNoResult:
|
|
retryCount++
|
|
if outcome.status == deletionOutcomeNoResult {
|
|
glog.Warningf("no deletion result for retried file %s, re-queuing to avoid loss", item.FileId)
|
|
}
|
|
// Keep in the durable ledger: still not confirmed.
|
|
f.DeletionRetryQueue.RequeueForRetry(item, outcome.errorMsg)
|
|
case deletionOutcomePermanent:
|
|
permanentErrorCount++
|
|
f.DeletionRetryQueue.Remove(item) // Remove from queue (permanent failure)
|
|
// gave up: drop the record, unless the id was re-queued meanwhile
|
|
// — a surviving newer record goes back through the hot queue.
|
|
if !f.forgetDeletionEpoch(item.FileId, item.ledgerEpoch) {
|
|
f.queueDeletions(false, item.FileId)
|
|
}
|
|
glog.Warningf("permanent error on retry for %s after %d attempts: %s", item.FileId, item.RetryCount, outcome.errorMsg)
|
|
}
|
|
}
|
|
|
|
if successCount > 0 || notFoundCount > 0 {
|
|
glog.V(1).Infof("retry: deleted %d files successfully, %d already deleted", successCount, notFoundCount)
|
|
}
|
|
if retryCount > 0 {
|
|
glog.V(1).Infof("retry: %d files still failing, will retry again later", retryCount)
|
|
}
|
|
if permanentErrorCount > 0 {
|
|
glog.Warningf("retry: %d files failed with permanent errors", permanentErrorCount)
|
|
}
|
|
}
|
|
|
|
func (f *Filer) DeleteUncommittedChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
|
|
f.doDeleteChunks(ctx, chunks, false)
|
|
}
|
|
|
|
func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks []*filer_pb.FileChunk) {
|
|
rule := f.FilerConf.MatchStorageRule(string(fullpath))
|
|
if rule.DisableChunkDeletion {
|
|
return
|
|
}
|
|
f.doDeleteChunks(ctx, chunks, rule.GetFsync())
|
|
}
|
|
|
|
func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk, fsync bool) {
|
|
for _, chunk := range chunks {
|
|
if !chunk.IsChunkManifest {
|
|
f.queueDeletions(fsync, chunk.GetFileIdString())
|
|
continue
|
|
}
|
|
dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk, f.MasterClient)
|
|
if manifestResolveErr != nil {
|
|
glog.V(0).InfofCtx(ctx, "failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr)
|
|
}
|
|
for _, dChunk := range dataChunks {
|
|
f.queueDeletions(fsync, dChunk.GetFileIdString())
|
|
}
|
|
f.queueDeletions(fsync, chunk.GetFileIdString())
|
|
}
|
|
}
|
|
|
|
func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) {
|
|
for _, chunk := range chunks {
|
|
f.queueDeletions(false, chunk.GetFileIdString())
|
|
}
|
|
}
|
|
|
|
func (f *Filer) deleteChunksIfNotNew(ctx context.Context, oldEntry, newEntry *Entry) {
|
|
var oldChunks, newChunks []*filer_pb.FileChunk
|
|
if oldEntry != nil {
|
|
oldChunks = oldEntry.GetChunks()
|
|
}
|
|
if newEntry != nil {
|
|
newChunks = newEntry.GetChunks()
|
|
}
|
|
|
|
toDelete, err := MinusChunks(ctx, f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks, f.MasterClient)
|
|
if err != nil {
|
|
glog.ErrorfCtx(ctx, "Failed to resolve old entry chunks when delete old entry chunks. new: %v, old: %v", filer_pb.FileIdsString(newChunks), filer_pb.FileIdsString(oldChunks))
|
|
return
|
|
}
|
|
f.DeleteChunksNotRecursive(toDelete)
|
|
}
|