package s3api import ( "context" "errors" "fmt" "strconv" "sync" "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // versioningHealLogPrefix is the grep-able tag every line in the .versions/ // pointer recovery lifecycle carries. Operators correlating produced / // surfaced / drained / gave-up events can pull a single stream across // hosts with `grep '\[versioning-heal\]'`. Keep this constant — log // aggregators (Loki, Splunk, journalctl filters) may match on it. const versioningHealLogPrefix = "[versioning-heal]" // versioningHealInfof / Warningf / Errorf emit a log line stamped with // the grep prefix and an event tag. Use event names from a small fixed // vocabulary so dashboards can aggregate by event: // // event=produced — stranded state was created (or would have been before this PR's fixes) // event=surfaced — read-path heal triggered for an already-stranded key // event=healed — pointer was successfully repaired or cleared // event=enqueue — reconciler queued a candidate // event=drain — reconciler successfully drained a candidate // event=retry — reconciler attempt failed, will retry // event=gave_up — reconciler exhausted retries on a candidate // event=anomaly — listing-after-rm inconsistency detected // event=summary — periodic heartbeat (counters + queue depth) // // All string/error arguments are sanitized so user-controlled bucket // names, object keys, filenames, and error text can't split the line // into multiple grep tokens or inject fake event=/bucket=/key= fields. // Safe values pass through unchanged; anything containing whitespace, // quotes, control chars, or backslashes is replaced with its // strconv.Quote form so the field boundary stays one space. func versioningHealInfof(event, format string, args ...interface{}) { glog.Infof("%s event=%s %s", versioningHealLogPrefix, event, fmt.Sprintf(format, sanitizeHealArgs(args)...)) } func versioningHealWarningf(event, format string, args ...interface{}) { glog.Warningf("%s event=%s %s", versioningHealLogPrefix, event, fmt.Sprintf(format, sanitizeHealArgs(args)...)) } func versioningHealErrorf(event, format string, args ...interface{}) { glog.Errorf("%s event=%s %s", versioningHealLogPrefix, event, fmt.Sprintf(format, sanitizeHealArgs(args)...)) } // sanitizeHealArgs walks args and replaces string/error values that // would break the single-space field separator (whitespace, newlines, // control chars, quotes, backslashes) with their strconv.Quote form. // Non-string args are returned untouched. func sanitizeHealArgs(args []interface{}) []interface{} { out := make([]interface{}, len(args)) for i, v := range args { out[i] = sanitizeHealArg(v) } return out } func sanitizeHealArg(v interface{}) interface{} { var s string switch t := v.(type) { case nil: return v case string: s = t case error: s = t.Error() default: return v } if !needsHealQuote(s) { return s } return strconv.Quote(s) } func needsHealQuote(s string) bool { for _, r := range s { if r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '"' || r == '\\' || r < 0x20 || r == 0x7f { return true } } return false } // Background reconciler for .versions/ directories whose latest-version // pointer references a missing file. The inline delete path // (deleteSpecificObjectVersion -> updateLatestVersionAfterDeletion) can // leave this state behind if the second step (pointer update / dir rm) // fails after the first (blob rm) succeeded. Read-path self-heal already // recovers on the next GET, but that surfaces NoSuchKey to the client. // The reconciler drains stranded entries proactively so a passing health // check or an idle key gets repaired without waiting for a client read. const ( versionsHealQueueCapacity = 4096 versionsHealPollInterval = 5 * time.Second versionsHealMaxRetries = 6 versionsHealBaseBackoff = 200 * time.Millisecond versionsHealMaxBackoff = 30 * time.Second ) type versionsHealCandidate struct { bucket string object string enqueued time.Time attempts int nextRetry time.Time } type versionsHealQueue struct { mu sync.Mutex pending map[string]*versionsHealCandidate } func newVersionsHealQueue() *versionsHealQueue { return &versionsHealQueue{ pending: make(map[string]*versionsHealCandidate), } } func versionsHealKey(bucket, object string) string { return bucket + "/" + object } // Enqueue records that bucket/object's .versions/ directory may be in a // stranded state and needs the reconciler to verify and heal it. Bounded // by versionsHealQueueCapacity so a stuck filer can't grow the map // without limit; on overflow we drop the newest candidate (the heal will // run on the next read or the next delete-failure for that key). func (q *versionsHealQueue) Enqueue(bucket, object string) { q.mu.Lock() defer q.mu.Unlock() key := versionsHealKey(bucket, object) if _, ok := q.pending[key]; ok { return } if len(q.pending) >= versionsHealQueueCapacity { versioningHealWarningf("queue_full", "bucket=%s key=%s capacity=%d (candidate dropped; read-path heal still covers it)", bucket, object, versionsHealQueueCapacity) return } q.pending[key] = &versionsHealCandidate{ bucket: bucket, object: object, enqueued: time.Now(), nextRetry: time.Now(), } versioningHealInfof("enqueue", "bucket=%s key=%s queue_depth=%d", bucket, object, len(q.pending)) } // popReady returns candidates whose nextRetry is in the past, removing // them from the queue. Callers reinsert via requeue() if the heal failed. func (q *versionsHealQueue) popReady(now time.Time) []*versionsHealCandidate { q.mu.Lock() defer q.mu.Unlock() var out []*versionsHealCandidate for k, c := range q.pending { if !c.nextRetry.After(now) { out = append(out, c) delete(q.pending, k) } } return out } // requeue re-inserts a candidate with an updated retry schedule. Drops // candidates that have hit the attempt cap so they don't loop forever // against a deterministically broken state; the read-path heal remains // as a last-resort safety net. func (q *versionsHealQueue) requeue(c *versionsHealCandidate, backoff time.Duration) { if c.attempts >= versionsHealMaxRetries { versioningHealWarningf("gave_up", "bucket=%s key=%s attempts=%d (read-path heal will still recover)", c.bucket, c.object, c.attempts) return } q.mu.Lock() defer q.mu.Unlock() c.nextRetry = time.Now().Add(backoff) q.pending[versionsHealKey(c.bucket, c.object)] = c } // Len returns the current queued size. Used in tests and by metrics. func (q *versionsHealQueue) Len() int { q.mu.Lock() defer q.mu.Unlock() return len(q.pending) } // startVersioningReconciler launches the background worker. Returns a // stop function that the server's Shutdown calls to cancel the loop. func (s3a *S3ApiServer) startVersioningReconciler() (stop func()) { if s3a.versionsHealQueue == nil { s3a.versionsHealQueue = newVersionsHealQueue() } ctx, cancel := context.WithCancel(context.Background()) go s3a.runVersioningReconciler(ctx) return cancel } func (s3a *S3ApiServer) runVersioningReconciler(ctx context.Context) { ticker := time.NewTicker(versionsHealPollInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: s3a.drainVersionsHealQueue() } } } func (s3a *S3ApiServer) drainVersionsHealQueue() { q := s3a.versionsHealQueue if q == nil { return } ready := q.popReady(time.Now()) for _, c := range ready { c.attempts++ if err := s3a.healVersionsPointer(c.bucket, c.object); err != nil { backoff := versionsHealBaseBackoff << uint(c.attempts-1) if backoff > versionsHealMaxBackoff { backoff = versionsHealMaxBackoff } versioningHealInfof("retry", "bucket=%s key=%s attempt=%d retry_in=%v err=%v", c.bucket, c.object, c.attempts, backoff, err) q.requeue(c, backoff) continue } versioningHealInfof("drain", "bucket=%s key=%s attempts=%d queued_for=%v", c.bucket, c.object, c.attempts, time.Since(c.enqueued)) } } // healVersionsPointer reads the .versions directory entry for the given // object and, if its latest-version pointer references a file that does // not exist, invokes the same self-heal path used by the read flow. A // genuinely-missing .versions directory or an already-consistent pointer // is treated as success — the candidate either healed itself via another // path or was never in trouble. // // Transient errors from the filer (timeout, brief unreachability) are // returned so the reconciler retries with backoff rather than silently // evicting the candidate. Swallowing them as "no stranded state" would // defeat the reconciler for exactly the failure modes the PR targets. func (s3a *S3ApiServer) healVersionsPointer(bucket, object string) error { bucketDir := s3a.bucketDir(bucket) versionsObjectPath := object + s3_constants.VersionsFolder versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath) if err != nil { if errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound { // Directory is gone — no stranded state to heal. return nil } return fmt.Errorf("read .versions entry %s/%s: %w", bucket, object, err) } if versionsEntry == nil || versionsEntry.Extended == nil { return nil } fileBytes, hasFile := versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey] if !hasFile || len(fileBytes) == 0 { // No pointer => no stranded state. return nil } latestFile := string(fileBytes) if _, probeErr := s3a.getEntry(bucketDir+"/"+versionsObjectPath, latestFile); probeErr == nil { // Pointer is consistent. return nil } else if !errors.Is(probeErr, filer_pb.ErrNotFound) && status.Code(probeErr) != codes.NotFound { // Transient — surface so the reconciler retries with backoff // rather than invoking heal on a possibly-incomplete listing // (which could rewrite the pointer to an older version). return fmt.Errorf("probe latest version %s/%s/%s: %w", bucket, object, latestFile, probeErr) } // Reuse the read-path heal so behaviour stays identical: it will // repair the pointer to the newest remaining version, or clear it // when nothing valid remains. _, healErr := s3a.healStaleLatestVersionPointer(bucket, object, versionsEntry, latestFile) return healErr }