seaweedfs/weed/pb/filer_pb_tail.go

package pb

import (
	"context"
	"errors"
	"fmt"
	"io"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

type EventErrorType int

const (
	TrivialOnError EventErrorType = iota
	FatalOnError
	RetryForeverOnError
	DontLogError
)

// MetadataFollowOption is used to control the behavior of the metadata following
// process. Part of it is used as a cursor to resume the following process.
type MetadataFollowOption struct {
	ClientName             string
	ClientId               int32
	ClientEpoch            int32
	SelfSignature          int32
	PathPrefix             string
	AdditionalPathPrefixes []string
	DirectoriesToWatch     []string
	StartTsNs              int64
	StopTsNs               int64
	EventErrorType         EventErrorType
	// LogFileReaderFn, when non-nil, enables metadata chunks mode:
	// the server sends log file chunk fids instead of streaming events,
	// and the client reads directly from volume servers.
	LogFileReaderFn LogFileReaderFn
	// OnIdleHeartbeat, when non-nil, opts in to idle heartbeats: while the
	// subscriber is caught up the server periodically sends an empty response
	// carrying the current time, and this is called with that timestamp. It is
	// a freshness signal only and does not advance StartTsNs, so the resume
	// checkpoint stays on the last real event.
	OnIdleHeartbeat func(tsNs int64)
	// GetResumeTsNs, when non-nil, supplies the reconnect position instead of
	// StartTsNs. It is read on every subscribe, so a callback can return the
	// durably processed watermark while StartTsNs keeps tracking positions the
	// stream has merely seen.
	GetResumeTsNs func() int64
	// Resubscribe, when closed, drops the stream with ErrResubscribe so the
	// caller's reconnect loop resubscribes from GetResumeTsNs and replays the
	// events still pinning the processed watermark. Target-side job failures
	// never surface on this stream, so without it a pinned event waits for an
	// unrelated source-stream reconnect (or a restart) to be replayed.
	Resubscribe <-chan struct{}
}

// ErrResubscribe ends a metadata follow when the consumer asked for the
// stream to drop so the reconnect replays what the resume watermark pins.
var ErrResubscribe = errors.New("resubscribe to replay events behind the failed offset")

type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error

func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {

	err := WithFilerClient(true, option.SelfSignature, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(option, processEventFn))
	if err != nil {
		return fmt.Errorf("subscribing filer meta change: %w", err)
	}
	return err
}

func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {

	err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(option, processEventFn))
	if err != nil {
		return fmt.Errorf("subscribing filer meta change: %w", err)
	}

	return nil
}

func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn ProcessMetadataFunc) func(client filer_pb.SeaweedFilerClient) error {
	return func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		sinceNs := option.StartTsNs
		if option.GetResumeTsNs != nil {
			sinceNs = option.GetResumeTsNs()
		}
		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName:                   option.ClientName,
			PathPrefix:                   option.PathPrefix,
			PathPrefixes:                 option.AdditionalPathPrefixes,
			Directories:                  option.DirectoriesToWatch,
			SinceNs:                      sinceNs,
			Signature:                    option.SelfSignature,
			ClientId:                     option.ClientId,
			ClientEpoch:                  option.ClientEpoch,
			UntilNs:                      option.StopTsNs,
			ClientSupportsBatching:       true,
			ClientSupportsMetadataChunks: option.LogFileReaderFn != nil,
			ClientSupportsIdleHeartbeat:  option.OnIdleHeartbeat != nil,
		})
		if err != nil {
			return fmt.Errorf("subscribe: %w", err)
		}

		if option.Resubscribe != nil {
			go func() {
				select {
				case <-option.Resubscribe:
					cancel()
				case <-ctx.Done():
				}
			}()
		}

		handleErr := func(resp *filer_pb.SubscribeMetadataResponse, err error) {
			sourcePath := filer_pb.MetadataEventSourceFullPath(resp)
			targetPath := filer_pb.MetadataEventTargetFullPath(resp)
			switch option.EventErrorType {
			case TrivialOnError:
				glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
			case FatalOnError:
				glog.Fatalf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
			case RetryForeverOnError:
				waitTime := time.Second
				for ctx.Err() == nil {
					if err := processEventFn(resp); err == nil {
						break
					} else {
						glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
					}
					select {
					case <-ctx.Done():
					case <-time.After(waitTime):
					}
					if waitTime < util.RetryWaitTime {
						waitTime += waitTime / 2
					}
				}
			case DontLogError:
				// pass
			default:
				glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
			}
		}

		// handleOneEvent processes a single response, whether it arrived as the
		// batch envelope or inside resp.Events. Freshness signals carry a timestamp
		// but no entry: the idle heartbeat (nil EventNotification) and the
		// MaxUnsyncedEvents marker (empty one). Both route to OnIdleHeartbeat,
		// else the marker pins sync_offset to the stale processed watermark and a
		// nil heartbeat folded into a batch nil-derefs in the sync job path.
		handleOneEvent := func(resp *filer_pb.SubscribeMetadataResponse) {
			if resp.EventNotification == nil || filer_pb.IsEmpty(resp) {
				if resp.TsNs > 0 && option.OnIdleHeartbeat != nil {
					option.OnIdleHeartbeat(resp.TsNs)
				}
				// The marker advances the resume cursor past the filtered range; the
				// heartbeat leaves it put so a restart cannot outrun a straggler. A
				// consumer with a resume callback keeps its cursor in its processed
				// watermark, so it sees the marker instead.
				if resp.EventNotification != nil && resp.TsNs > 0 {
					if option.GetResumeTsNs != nil {
						if err := processEventFn(resp); err != nil {
							handleErr(resp, err)
						}
					} else {
						option.StartTsNs = resp.TsNs
					}
				}
				return
			}
			if err := processEventFn(resp); err != nil {
				handleErr(resp, err)
				// RetryForeverOnError only returns once the event was handled;
				// other modes leave it failed and the cursor stays behind it.
				if option.EventErrorType != RetryForeverOnError {
					return
				}
			}
			if option.GetResumeTsNs == nil {
				option.StartTsNs = resp.TsNs
			}
		}

		var pendingRefs []*filer_pb.LogFileChunkRef

		// drainPendingRefs reads whatever chunk refs have accumulated. The server
		// sends refs in their own responses, so they can only be read once
		// something tells us the run of refs has ended — either a normal event, or
		// the end of the stream. A bounded subscription (StopTsNs set, range
		// already in the past) may get nothing but refs and then EOF, so draining
		// only on the former silently returns no events at all.
		drainPendingRefs := func() error {
			if len(pendingRefs) == 0 || option.LogFileReaderFn == nil {
				return nil
			}
			readFromNs := option.StartTsNs
			var progressFn func(tsNs int64)
			if option.GetResumeTsNs != nil {
				// The resume cursor lives in the processed watermark, so a
				// resubscribed ref replay must not be filtered by positions the
				// previous stream had only seen.
				readFromNs = sinceNs
			} else {
				// Checkpoint each settled entry so a dropped stream resumes the
				// replay where it stopped instead of re-reading the whole set.
				progressFn = func(tsNs int64) {
					if tsNs > option.StartTsNs {
						option.StartTsNs = tsNs
					}
				}
			}
			lastTs, readErr := ReadLogFileRefs(pendingRefs, option.LogFileReaderFn,
				readFromNs, option.StopTsNs,
				PathFilter{
					PathPrefix:             option.PathPrefix,
					AdditionalPathPrefixes: option.AdditionalPathPrefixes,
					DirectoriesToWatch:     option.DirectoriesToWatch,
				},
				processEventFn, progressFn)
			if readErr != nil {
				return fmt.Errorf("%w: %w", ErrLogFileRead, readErr)
			}
			// lastTs is the last settled entry's own timestamp; never rewind
			// below the progress checkpoint it may lag.
			if option.GetResumeTsNs == nil && lastTs > option.StartTsNs {
				option.StartTsNs = lastTs
			}
			pendingRefs = nil
			return nil
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return drainPendingRefs()
			}
			if listenErr != nil {
				select {
				case <-option.Resubscribe:
					return ErrResubscribe
				default:
				}
				return listenErr
			}

			// Accumulate log file chunk references (metadata chunks mode)
			if len(resp.LogFileRefs) > 0 {
				pendingRefs = append(pendingRefs, resp.LogFileRefs...)
				continue
			}

			// Process accumulated refs before handling normal events (transition point)
			if err := drainPendingRefs(); err != nil {
				return err
			}

			// Process the envelope event (top-level fields) and any batched tail.
			// The server folds a backlog into one response: the first event lives in
			// the top-level fields, the rest in resp.Events. Either slot can hold a
			// freshness signal, so both go through the same handler.
			handleOneEvent(resp)
			for _, batchedEvent := range resp.Events {
				handleOneEvent(batchedEvent)
			}
		}
	}
}

func AddOffsetFunc(processEventFn ProcessMetadataFunc, offsetInterval time.Duration, offsetFunc func(counter int64, offset int64) error) ProcessMetadataFunc {
	var counter int64
	var lastWriteTime = time.Now()
	return func(resp *filer_pb.SubscribeMetadataResponse) error {
		if err := processEventFn(resp); err != nil {
			return err
		}
		counter++
		if lastWriteTime.Add(offsetInterval).Before(time.Now()) {
			lastWriteTime = time.Now()
			if err := offsetFunc(counter, resp.TsNs); err != nil {
				return err
			}
			counter = 0
		}
		return nil
	}

}