seaweedfs/weed/s3api/s3lifecycle/scheduler/configload.go

package scheduler

import (
	"context"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/lifecycle_xml"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
)

// The bucket lifecycle configuration lives here rather than in weed/s3api so
// callers outside the gateway (the admin dashboard, the scheduler itself) and
// the gateway share one definition of the extended-attribute keys and the
// size cap. weed/s3api aliases these; nothing redeclares the values.
const (
	BucketLifecycleConfigurationXMLKey            = "s3-bucket-lifecycle-configuration-xml"
	BucketLifecycleTransitionMinimumObjectSizeKey = "s3-bucket-lifecycle-transition-default-minimum-object-size"

	// MaxBucketLifecycleConfigurationSize caps the serialized XML, mirroring
	// the S3 API's limit.
	MaxBucketLifecycleConfigurationSize = 1 << 20
)

// ParseError is returned alongside successfully-loaded inputs so callers can
// surface malformed bucket configs rather than silently dropping them.
type ParseError struct {
	Bucket string
	Err    error
}

// LoadCompileInputs walks the buckets directory and returns one
// engine.CompileInput per bucket that carries a non-empty lifecycle
// configuration. Pagination loops with startFrom so clusters with more
// than one page of buckets don't drop the tail.
func LoadCompileInputs(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketsPath string) ([]engine.CompileInput, []ParseError, error) {
	var (
		inputs      []engine.CompileInput
		parseErrors []ParseError
		startFrom   string
	)
	const pageSize uint32 = 1024
	for {
		pageCount := 0
		var lastName string
		err := filer_pb.SeaweedList(ctx, client, bucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
			pageCount++
			lastName = entry.Name
			if !entry.IsDirectory {
				return nil
			}
			xmlBytes, ok := entry.Extended[BucketLifecycleConfigurationXMLKey]
			if !ok || len(xmlBytes) == 0 {
				return nil
			}
			rules, err := lifecycle_xml.ParseCanonical(xmlBytes)
			if err != nil {
				parseErrors = append(parseErrors, ParseError{Bucket: entry.Name, Err: err})
				return nil
			}
			if len(rules) == 0 {
				return nil
			}
			inputs = append(inputs, engine.CompileInput{
				Bucket:    entry.Name,
				Rules:     rules,
				Versioned: IsBucketVersioned(entry),
			})
			return nil
		}, startFrom, false, pageSize)
		if err != nil {
			return nil, nil, err
		}
		if uint32(pageCount) < pageSize {
			break
		}
		startFrom = lastName
	}
	return inputs, parseErrors, nil
}

// IsBucketVersioned returns true when the bucket's filer entry has the
// versioning extended attribute set to Enabled or Suspended, or when
// Object Lock is on: lock requires versioning, so the gateway writes
// version siblings even if the versioning attribute is absent.
func IsBucketVersioned(entry *filer_pb.Entry) bool {
	if v, ok := entry.Extended[s3_constants.ExtVersioningKey]; ok {
		s := strings.ToLower(strings.TrimSpace(string(v)))
		if s == "enabled" || s == "suspended" {
			return true
		}
	}
	return string(entry.Extended[s3_constants.ExtObjectLockEnabledKey]) == s3_constants.ObjectLockEnabled
}

// AllActivePriorStates seeds every compiled action as bootstrap-complete +
// event-driven so the scheduler dispatches whatever fires immediately.
// Production bootstrap walks set this incrementally per bucket; the
// scheduler runs out-of-band of that flow for now.
func AllActivePriorStates(inputs []engine.CompileInput) map[s3lifecycle.ActionKey]engine.PriorState {
	prior := map[s3lifecycle.ActionKey]engine.PriorState{}
	for _, in := range inputs {
		for _, rule := range in.Rules {
			hash := s3lifecycle.RuleHash(rule)
			for _, kind := range s3lifecycle.RuleActionKinds(rule) {
				key := s3lifecycle.ActionKey{Bucket: in.Bucket, RuleHash: hash, ActionKind: kind}
				prior[key] = engine.PriorState{
					BootstrapComplete: true,
					Mode:              engine.ModeEventDriven,
				}
			}
		}
	}
	return prior
}