s3 lifecycle: route version-file writes to current-version expiration (#11956)

* s3lifecycle: schedule ExpirationDays from version-file writes

On a versioned bucket a PUT lands in the meta-log as the version file
plus the .versions/ directory update — there is no bare-key event, so
the version-folder skip dropped every version-file event for all kinds
but ExpiredDeleteMarker. ExpirationDays therefore only fired via the
recovery walker's cold-start / rule-change pass: once a shard had a
saved cursor, no daily run ever scheduled another delete marker.

A version-file write is the latest for its logical key at write time,
so route it through the same event-driven evaluation the regular path
runs. The match carries the file's identity with an empty VersionID —
the dispatcher resolves the current latest at due time, and a fresher
PUT meanwhile resolves to STALE_IDENTITY there.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* lifecycle test: versioned Expiration fires on the second daily run

Establishes the shard cursor with a first pass, then PUTs and backdates
a new versioned object and runs again. The delete marker must appear
without a rule change forcing a recovery walk.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: run TestLifecycleVersionedBucketExpiresOnSecondRun in the lifecycle matrix

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/s3-go-tests.yml

@@ -283,6 +283,7 @@ jobs:
           - TestLifecycleSuspendedVersioningExpiration
           - TestLifecycleTagFilter
           - TestLifecycleVersionedBucketCreatesDeleteMarker
+          - TestLifecycleVersionedBucketExpiresOnSecondRun
 
     steps:
       - name: Check out code

test/s3/lifecycle/s3_lifecycle_versioning_test.go

@@ -164,6 +164,55 @@ func TestLifecycleVersionedBucketCreatesDeleteMarker(t *testing.T) {
 	require.True(t, sawMarker, "ListObjectVersions must show a delete marker as latest for %s", key)
 }
 
+// TestLifecycleVersionedBucketExpiresOnSecondRun covers the reported
+// regression where Expiration.Days only fired on the cold-start walk: a
+// versioned PUT surfaces in the meta-log as a version-file event, which
+// the router used to drop, so once every shard had a saved cursor no
+// daily run ever scheduled another delete marker. The first run here
+// establishes the cursor; the second must expire an object that arrived
+// in between, with no rule change to force a recovery walk.
+func TestLifecycleVersionedBucketExpiresOnSecondRun(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("vexp2")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+
+	// First pass: any object will do — what matters is the cursor saved
+	// afterward, which switches later passes from the recovery walk to
+	// meta-log replay.
+	putObject(t, c, bucket, "expire/first.txt", "v1")
+	runLifecycleShard(t)
+
+	const key = "expire/daily.txt"
+	putObject(t, c, bucket, key, "v1")
+	headOut, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+	versionID := aws.ToString(headOut.VersionId)
+	require.NotEmpty(t, versionID)
+	backdateVersionedMtime(t, fc, bucket, key, versionID, 30)
+
+	runLifecycleShard(t)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "second run must create a delete marker for %s/%s", bucket, key)
+
+	directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(versionID),
+	})
+	require.NoError(t, err, "original version must still exist")
+	require.NotNil(t, directHead)
+}
+
 // TestLifecycleNoncurrentVersionExpiration: NoncurrentVersionExpiration
 // only fires on noncurrent versions. PUT v1, PUT v2 (so v1 → noncurrent),
 // backdate v1, run worker. v1 must be removed; v2 stays current.

weed/s3api/s3lifecycle/router/router.go

@@ -109,11 +109,19 @@ func Route(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, now tim
 	// EXP_DM can fire on two version-folder events: the marker create
 	// (sole survivor immediately) and a noncurrent hard-delete that
 	// leaves only the marker behind. Both reach routeSoleSurvivorMarker.
+	// A stamped version-file write is also the latest for its logical key
+	// at write time — the versioned layout emits no bare-key event the
+	// regular path could see, so ExpirationDays must schedule here or it
+	// only ever fires via a cold-start / rule-change recovery walk.
 	if versioned && isVersionFolderPath(ev.Key) {
-		if !hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) {
-			return nil
+		var matches []Match
+		if isVersionFileWrite(ev.NewEntry) {
+			matches = routeVersionFileExpiration(snap, ev, keys)
 		}
-		return routeSoleSurvivorMarker(ctx, snap, ev, keys, lister)
+		if hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) {
+			matches = append(matches, routeSoleSurvivorMarker(ctx, snap, ev, keys, lister)...)
+		}
+		return matches
 	}
 
 	if ev.NewEntry == nil {
@@ -517,6 +525,71 @@ func emitNoncurrentMatches(snap *engine.Snapshot, ev *reader.Event, keys []s3lif
 	return matches
 }
 
+// isVersionFileWrite reports whether the event's new entry is a genuine
+// version-file write: a non-marker file carrying ExtVersionIdKey. The
+// version id stamp is what separates it from a literal user file under a
+// *.versions/ parent.
+func isVersionFileWrite(entry *filer_pb.Entry) bool {
+	if entry == nil || entry.IsDirectory || isDeleteMarkerEntry(entry) {
+		return false
+	}
+	return len(entry.Extended[s3_constants.ExtVersionIdKey]) > 0
+}
+
+// routeVersionFileExpiration schedules current-version expiration from a
+// version-file write. The file is the latest at write time, and VersionID
+// stays empty so the dispatcher resolves the latest at due time — a
+// fresher PUT meanwhile resolves to STALE_IDENTITY there, and that PUT's
+// own event carried its own schedule. A write that turns out to be an
+// older version's metadata update no-ops the same way.
+func routeVersionFileExpiration(snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey) []Match {
+	logical, ok := logicalKeyFromVersionPath(ev.Key)
+	if !ok {
+		return nil
+	}
+	entry := ev.NewEntry
+	if entry.Attributes == nil {
+		return nil
+	}
+	info := &s3lifecycle.ObjectInfo{
+		Key:      logical,
+		ModTime:  time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)),
+		Size:     int64(entry.Attributes.FileSize),
+		IsLatest: true,
+	}
+	if tags := extractTags(entry.Extended); len(tags) > 0 {
+		info.Tags = tags
+	}
+	eventTime := time.Unix(0, ev.TsNs)
+	identity := buildIdentityFromEntry(entry)
+	var matches []Match
+	for _, key := range keys {
+		action := snap.Action(key)
+		if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven {
+			continue
+		}
+		dueTime := s3lifecycle.ComputeDueAt(action.Rule, key.ActionKind, info)
+		if dueTime.IsZero() {
+			continue
+		}
+		res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime)
+		if res.Action == s3lifecycle.ActionNone {
+			continue
+		}
+		matches = append(matches, Match{
+			Key:       key,
+			Action:    action,
+			Result:    res,
+			EventTs:   eventTime,
+			DueTime:   dueTime,
+			Bucket:    ev.Bucket,
+			ObjectKey: logical,
+			Identity:  identity,
+		})
+	}
+	return matches
+}
+
 // routeSoleSurvivorMarker emits an EXP_DM Match against the LOGICAL key
 // (so the dispatcher can call deleteSpecificObjectVersion) with the
 // marker's version_id. Handles two events: a marker create (the new

weed/s3api/s3lifecycle/router/router_test.go

@@ -458,6 +458,65 @@ func TestRouteVersionedCurrentEventStaysLatest(t *testing.T) {
 	}
 }
 
+func TestRouteVersionedVersionFileWriteSchedulesExpiration(t *testing.T) {
+	// A versioned PUT shows up in the meta-log as the version file plus
+	// the .versions/ directory update — there is no bare-key event for
+	// the regular path to see. The version file is the latest at write
+	// time; the match carries its identity with an empty VersionID so
+	// dispatch CAS-checks the current latest at due time.
+	rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
+	snap := compileWithVersioned(rule, activatedPrior(rule))
+
+	now := time.Now()
+	old := now.AddDate(0, 0, -2)
+	ev := eventCreate("bk", "logs/foo"+s3_constants.VersionsFolder+"/v_v1", old.Unix(), 1, old.UnixNano())
+	ev.NewEntry.Extended = map[string][]byte{
+		s3_constants.ExtVersionIdKey: []byte("v1"),
+	}
+
+	matches := Route(context.Background(), snap, ev, now, nil)
+	if len(matches) != 1 {
+		t.Fatalf("expected 1 match (EXPIRATION_DAYS), got %v", matches)
+	}
+	m := matches[0]
+	if m.Key.ActionKind != s3lifecycle.ActionKindExpirationDays {
+		t.Fatalf("ActionKind=%v, want ExpirationDays", m.Key.ActionKind)
+	}
+	if m.ObjectKey != "logs/foo" {
+		t.Fatalf("ObjectKey=%q, want the logical key", m.ObjectKey)
+	}
+	if m.VersionID != "" {
+		t.Fatalf("VersionID=%q, want empty so dispatch resolves the current latest", m.VersionID)
+	}
+	if m.Identity == nil {
+		t.Fatal("Identity must fingerprint the version file for dispatch CAS")
+	}
+	if want := time.Unix(old.Unix(), 0).Add(s3lifecycle.DaysToDuration(1)); !m.DueTime.Equal(want) {
+		t.Fatalf("DueTime=%v, want %v", m.DueTime, want)
+	}
+}
+
+func TestRouteVersionedMarkerCreateDoesNotFireExpiration(t *testing.T) {
+	// A delete marker is a version file too, but ExpirationDays never
+	// applies to a marker — only EXP_DM, handled by the sole-survivor
+	// path. Without the marker gate, the expiration match would pin a
+	// marker identity and dispatch would stack a second marker on top.
+	rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
+	snap := compileWithVersioned(rule, activatedPrior(rule))
+
+	now := time.Now()
+	old := now.AddDate(0, 0, -2)
+	ev := markerEvent("bk", "logs/gone", "2026-05-09-abc", old.Unix(), old.UnixNano())
+
+	lister := &recordingLister{survivors: Survivors{Count: 1}}
+	if got := Route(context.Background(), snap, ev, now, lister); len(got) != 0 {
+		t.Fatalf("marker create must not fire EXPIRATION_DAYS, got %v", got)
+	}
+	if len(lister.calls) != 0 {
+		t.Fatalf("lister consulted without an EXP_DM rule: calls=%v", lister.calls)
+	}
+}
+
 func TestRouteNonVersionedBucketIgnoresVersionsSuffix(t *testing.T) {
 	// A non-versioned bucket happens to have an object literally named
 	// "logs/foo.versions/v1" — that's just a regular path. The router
@@ -796,13 +855,11 @@ func TestRouteVersionedDeleteMarkerNoExpDMRuleSkipsListing(t *testing.T) {
 }
 
 func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) {
-	// On a versioned bucket the router skips every event whose parent
-	// directory name ends with ".versions" — both the version files
-	// SeaweedFS itself writes (logs/foo.versions/v_v1) and any literal
-	// key the user happens to put under such a parent — because the
-	// current-vs-noncurrent classification needs the .versions/
-	// directory's latest pointer, which isn't carried by these events.
-	// Bootstrap covers retention for those entries.
+	// On a versioned bucket, events inside a *.versions/ parent that are
+	// not stamped version-file writes stay unrouted — current-vs-noncurrent
+	// classification needs the container's latest pointer, and the file's
+	// own ExtVersionIdKey is what marks it as a version write the
+	// expiration path can schedule from.
 	rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
 	snap := compileWithVersioned(rule, activatedPrior(rule))
 
@@ -815,9 +872,11 @@ func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) {
 		ext   map[string][]byte
 	}{
 		{
-			name: "tracked version file",
+			// A marker carries the version id but is a marker, not an
+			// expirable write — it routes to the sole-survivor path.
+			name: "delete marker",
 			key:  "logs/foo.versions/v_v1",
-			ext:  map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")},
+			ext:  map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1"), s3_constants.ExtDeleteMarkerKey: []byte("true")},
 		},
 		{
 			name: "literal-key collision",