s3_lifecycle: bound shared metadata subscriptions

weed/s3api/s3lifecycle/reader/reader.go

@@ -73,8 +73,17 @@ type Reader struct {
 	// then means "subscribe from the start of the meta-log".
 	Cursor    *Cursor
 	StartTsNs int64
+	// UntilTsNs bounds the subscription at an inclusive metadata timestamp.
+	// Zero leaves the stream unbounded.
+	UntilTsNs int64
 	Events    chan<- *Event
 
+	// ReceiveTimeout bounds the time spent waiting for the next stream
+	// response. The filer sends idle heartbeats to readers that opt in, so a
+	// caught-up but healthy stream remains active while a half-open stream
+	// eventually fails. Zero disables the timeout.
+	ReceiveTimeout time.Duration
+
 	// EventBudget caps how many events Run processes before returning nil.
 	// Zero = unbounded; the run continues until ctx cancellation or stream
 	// error. Used by the worker scheduler to bound a single READ task.
@@ -102,6 +111,9 @@ func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, cl
 	if r.BucketsPath == "" {
 		return errors.New("reader: empty BucketsPath")
 	}
+	if r.ReceiveTimeout < 0 {
+		return fmt.Errorf("reader: negative ReceiveTimeout %v", r.ReceiveTimeout)
+	}
 	r.bucketsPathSlash = r.BucketsPath
 	if !strings.HasSuffix(r.bucketsPathSlash, "/") {
 		r.bucketsPathSlash += "/"
@@ -111,20 +123,70 @@ func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, cl
 	if sinceNs == 0 && r.Cursor != nil {
 		sinceNs = r.Cursor.MinTsNs()
 	}
-	stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
+	streamCtx, cancelStream := context.WithCancel(ctx)
+	defer cancelStream()
+	stream, err := client.SubscribeMetadata(streamCtx, &filer_pb.SubscribeMetadataRequest{
 		ClientName:             clientName,
 		PathPrefix:             r.BucketsPath,
 		SinceNs:                sinceNs,
 		ClientId:               clientID,
+		UntilNs:                r.UntilTsNs,
 		ClientSupportsBatching: true,
+		// Heartbeats provide application-level proof that the response path is
+		// alive. They are consumed below like any other response but filtered
+		// out by dispatchOne because they carry no EventNotification.
+		ClientSupportsIdleHeartbeat: r.ReceiveTimeout > 0,
 	})
 	if err != nil {
 		return fmt.Errorf("subscribe: %w", err)
 	}
 
+	type receiveResult struct {
+		response *filer_pb.SubscribeMetadataResponse
+		err      error
+	}
+	receiveCh := make(chan receiveResult, 1)
+	go func() {
+		for {
+			resp, recvErr := stream.Recv()
+			select {
+			case receiveCh <- receiveResult{response: resp, err: recvErr}:
+			case <-streamCtx.Done():
+				return
+			}
+			if recvErr != nil {
+				return
+			}
+		}
+	}()
+
 	processed := 0
 	for {
-		resp, recvErr := stream.Recv()
+		var (
+			resp    *filer_pb.SubscribeMetadataResponse
+			recvErr error
+			timer   *time.Timer
+			timeout <-chan time.Time
+		)
+		if r.ReceiveTimeout > 0 {
+			timer = time.NewTimer(r.ReceiveTimeout)
+			timeout = timer.C
+		}
+		select {
+		case result := <-receiveCh:
+			if timer != nil {
+				timer.Stop()
+			}
+			resp, recvErr = result.response, result.err
+		case <-timeout:
+			cancelStream()
+			return fmt.Errorf("%w after %s", ErrReceiveTimeout, r.ReceiveTimeout)
+		case <-streamCtx.Done():
+			if timer != nil {
+				timer.Stop()
+			}
+			return streamCtx.Err()
+		}
 		if recvErr == io.EOF {
 			return nil
 		}
@@ -147,6 +209,10 @@ func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, cl
 	}
 }
 
+// ErrReceiveTimeout reports that an otherwise open subscription stopped
+// delivering both metadata events and negotiated idle heartbeats.
+var ErrReceiveTimeout = errors.New("reader: metadata receive timeout")
+
 func (r *Reader) dispatchOne(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, processed *int) error {
 	if resp == nil || resp.EventNotification == nil {
 		return nil

weed/s3api/s3lifecycle/reader/receive_timeout_test.go

@@ -0,0 +1,51 @@
+package reader
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+type blockingSubscribeClient struct {
+	filer_pb.SeaweedFilerClient
+	request *filer_pb.SubscribeMetadataRequest
+}
+
+func (c *blockingSubscribeClient) SubscribeMetadata(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.SubscribeMetadataResponse], error) {
+	c.request = req
+	return &blockingSubscribeStream{ctx: ctx}, nil
+}
+
+type blockingSubscribeStream struct {
+	grpc.ServerStreamingClient[filer_pb.SubscribeMetadataResponse]
+	ctx context.Context
+}
+
+func (s *blockingSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	<-s.ctx.Done()
+	return nil, s.ctx.Err()
+}
+
+func TestRunTimesOutHalfOpenSubscription(t *testing.T) {
+	client := &blockingSubscribeClient{}
+	r := &Reader{
+		ShardID:        0,
+		BucketsPath:    "/buckets",
+		UntilTsNs:      12345,
+		Events:         make(chan *Event),
+		ReceiveTimeout: 50 * time.Millisecond,
+	}
+
+	started := time.Now()
+	err := r.Run(context.Background(), client, "test-lifecycle", 7)
+	require.ErrorIs(t, err, ErrReceiveTimeout)
+	assert.Less(t, time.Since(started), time.Second)
+	require.NotNil(t, client.request)
+	assert.Equal(t, int64(12345), client.request.UntilNs)
+	assert.True(t, client.request.ClientSupportsIdleHeartbeat)
+}