seaweedfs/weed/worker/tasks/iceberg/config.go

package iceberg

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)

const (
	jobType = "iceberg_maintenance"

	defaultSnapshotRetentionHours = 168 // 7 days
	defaultMaxSnapshotsToKeep     = 5
	defaultOrphanOlderThanHours   = 72
	defaultMaxCommitRetries       = 5
	defaultTargetFileSizeMB       = 256
	defaultMinInputFiles          = 5
	defaultDeleteTargetFileSizeMB = 64
	defaultDeleteMinInputFiles    = 2
	defaultDeleteMaxGroupSizeMB   = 256
	defaultDeleteMaxOutputFiles   = 8
	defaultRewriteStrategy        = "binpack"
	rewriteStrategyAuto           = "auto"
	defaultSortBufferRows         = 262144
	minSortBufferRows             = 1024
	defaultMinManifestsToRewrite  = 5
	minManifestsToRewrite         = 2
	defaultOperations             = "all"

	// Metric keys returned by maintenance operations.
	MetricFilesMerged          = "files_merged"
	MetricFilesWritten         = "files_written"
	MetricBins                 = "bins"
	MetricSnapshotsExpired     = "snapshots_expired"
	MetricFilesDeleted         = "files_deleted"
	MetricOrphansRemoved       = "orphans_removed"
	MetricManifestsRewritten   = "manifests_rewritten"
	MetricDeleteFilesRewritten = "delete_files_rewritten"
	MetricDeleteFilesWritten   = "delete_files_written"
	MetricDeleteBytesRewritten = "delete_bytes_rewritten"
	MetricDeleteGroupsPlanned  = "delete_groups_planned"
	MetricDeleteGroupsSkipped  = "delete_groups_skipped"
	MetricEntriesTotal         = "entries_total"
	MetricDurationMs           = "duration_ms"
)

const bytesPerMB int64 = 1024 * 1024

const msPerHour int64 = 3600 * 1000

// maxOrphanOlderThanHours is the largest cutoff that still survives conversion
// to a time.Duration. Beyond it the multiplication wraps negative, putting the
// cutoff in the future and making every file look like an orphan.
const maxOrphanOlderThanHours = int64(math.MaxInt64 / int64(time.Hour))

// mbToBytes converts megabytes to bytes, saturating instead of overflowing.
func mbToBytes(mb int64) int64 {
	if mb <= 0 {
		return 0
	}
	if mb > math.MaxInt64/bytesPerMB {
		return math.MaxInt64
	}
	return mb * bytesPerMB
}

// addDays sums two day counts, saturating instead of overflowing. Negative
// inputs are treated as unset.
func addDays(a, b int64) int64 {
	if a < 0 {
		a = 0
	}
	if b < 0 {
		b = 0
	}
	if a > math.MaxInt64-b {
		return math.MaxInt64
	}
	return a + b
}

// daysToHours converts days to hours, saturating instead of overflowing.
// applyThresholdDefaults clamps the result down to a usable cutoff.
func daysToHours(days int64) int64 {
	if days <= 0 {
		return 0
	}
	if days > math.MaxInt64/24 {
		return math.MaxInt64
	}
	return days * 24
}

// hoursToMs converts hours to milliseconds, saturating instead of overflowing.
func hoursToMs(hours int64) int64 {
	if hours <= 0 {
		return 0
	}
	if hours > math.MaxInt64/msPerHour {
		return math.MaxInt64
	}
	return hours * msPerHour
}

// Config holds parsed worker config values.
type Config struct {
	// Milliseconds rather than hours so sub-hour retentions survive; the
	// plugin config key stays in hours.
	SnapshotRetentionMs         int64
	MaxSnapshotsToKeep          int64
	OrphanOlderThanHours        int64
	MaxCommitRetries            int64
	TargetFileSizeBytes         int64
	MinInputFiles               int64
	DeleteTargetFileSizeBytes   int64
	DeleteMinInputFiles         int64
	DeleteMaxFileGroupSizeBytes int64
	DeleteMaxOutputFiles        int64
	MinManifestsToRewrite       int64
	Operations                  string
	TablePropertiesOverride     bool
	ApplyDeletes                bool
	Where                       string
	RewriteStrategy             string
	SortMaxInputBytes           int64
	SortBufferRows              int64
	SortSpillDir                string
	// Row groups of a compacted file, from the table's write.parquet.*
	// properties; zero means the default.
	RowGroupSizeBytes int64
	RowGroupRowLimit  int64
}

// ParseConfig extracts an iceberg maintenance Config from plugin config values.
// Values are clamped to safe minimums to prevent misconfiguration.
func ParseConfig(values map[string]*plugin_pb.ConfigValue) Config {
	cfg := Config{
		SnapshotRetentionMs:         hoursToMs(readInt64Config(values, "snapshot_retention_hours", defaultSnapshotRetentionHours)),
		MaxSnapshotsToKeep:          readInt64Config(values, "max_snapshots_to_keep", defaultMaxSnapshotsToKeep),
		OrphanOlderThanHours:        readInt64Config(values, "orphan_older_than_hours", defaultOrphanOlderThanHours),
		MaxCommitRetries:            readInt64Config(values, "max_commit_retries", defaultMaxCommitRetries),
		TargetFileSizeBytes:         readSizeMBConfig(values, "target_file_size_mb", defaultTargetFileSizeMB),
		MinInputFiles:               readInt64Config(values, "min_input_files", defaultMinInputFiles),
		DeleteTargetFileSizeBytes:   readSizeMBConfig(values, "delete_target_file_size_mb", defaultDeleteTargetFileSizeMB),
		DeleteMinInputFiles:         readInt64Config(values, "delete_min_input_files", defaultDeleteMinInputFiles),
		DeleteMaxFileGroupSizeBytes: readSizeMBConfig(values, "delete_max_file_group_size_mb", defaultDeleteMaxGroupSizeMB),
		DeleteMaxOutputFiles:        readInt64Config(values, "delete_max_output_files", defaultDeleteMaxOutputFiles),
		MinManifestsToRewrite:       readInt64Config(values, "min_manifests_to_rewrite", defaultMinManifestsToRewrite),
		Operations:                  readStringConfig(values, "operations", defaultOperations),
		TablePropertiesOverride:     readBoolConfig(values, "table_properties_override", true),
		ApplyDeletes:                readBoolConfig(values, "apply_deletes", true),
		Where:                       strings.TrimSpace(readStringConfig(values, "where", "")),
		RewriteStrategy:             strings.TrimSpace(strings.ToLower(readStringConfig(values, "rewrite_strategy", defaultRewriteStrategy))),
		SortMaxInputBytes:           readSizeMBConfig(values, "sort_max_input_mb", 0),
		SortBufferRows:              readInt64Config(values, "sort_buffer_rows", defaultSortBufferRows),
		SortSpillDir:                strings.TrimSpace(readStringConfig(values, "sort_spill_dir", "")),
	}

	// Clamp the fields that are always defaulted by worker config parsing.
	if cfg.SnapshotRetentionMs <= 0 {
		cfg.SnapshotRetentionMs = hoursToMs(defaultSnapshotRetentionHours)
	}
	if cfg.MaxSnapshotsToKeep <= 0 {
		cfg.MaxSnapshotsToKeep = defaultMaxSnapshotsToKeep
	}
	if cfg.MaxCommitRetries <= 0 {
		cfg.MaxCommitRetries = defaultMaxCommitRetries
	}
	cfg = applyThresholdDefaults(cfg)
	return cfg
}

func applyThresholdDefaults(cfg Config) Config {
	if cfg.OrphanOlderThanHours <= 0 {
		cfg.OrphanOlderThanHours = defaultOrphanOlderThanHours
	}
	if cfg.OrphanOlderThanHours > maxOrphanOlderThanHours {
		cfg.OrphanOlderThanHours = maxOrphanOlderThanHours
	}
	if cfg.TargetFileSizeBytes <= 0 {
		cfg.TargetFileSizeBytes = defaultTargetFileSizeMB * 1024 * 1024
	}
	if cfg.MinInputFiles < 2 {
		cfg.MinInputFiles = defaultMinInputFiles
	}
	if cfg.DeleteTargetFileSizeBytes <= 0 {
		cfg.DeleteTargetFileSizeBytes = defaultDeleteTargetFileSizeMB * 1024 * 1024
	}
	if cfg.DeleteMinInputFiles < 2 {
		cfg.DeleteMinInputFiles = defaultDeleteMinInputFiles
	}
	if cfg.DeleteMaxFileGroupSizeBytes <= 0 {
		cfg.DeleteMaxFileGroupSizeBytes = defaultDeleteMaxGroupSizeMB * 1024 * 1024
	}
	if cfg.DeleteMaxOutputFiles <= 0 {
		cfg.DeleteMaxOutputFiles = defaultDeleteMaxOutputFiles
	}
	if cfg.RewriteStrategy == "" {
		cfg.RewriteStrategy = defaultRewriteStrategy
	}
	if cfg.RewriteStrategy != "binpack" && cfg.RewriteStrategy != "sort" && cfg.RewriteStrategy != rewriteStrategyAuto {
		cfg.RewriteStrategy = defaultRewriteStrategy
	}
	if cfg.SortMaxInputBytes < 0 {
		cfg.SortMaxInputBytes = 0
	}
	// A run smaller than this buys no memory back worth the extra runs, and
	// the writer stops at 32K of them, so a tiny value would cap the output.
	if cfg.SortBufferRows < minSortBufferRows {
		cfg.SortBufferRows = defaultSortBufferRows
	}
	if cfg.MinManifestsToRewrite < minManifestsToRewrite {
		cfg.MinManifestsToRewrite = minManifestsToRewrite
	}
	return cfg
}

func readSizeMBConfig(values map[string]*plugin_pb.ConfigValue, field string, fallbackMB int64) int64 {
	mb := readInt64Config(values, field, fallbackMB)
	if mb <= 0 {
		return 0
	}
	maxMB := int64(^uint64(0)>>1) / bytesPerMB
	if mb > maxMB {
		glog.V(1).Infof("readSizeMBConfig: clamping %q from %d MB to %d MB", field, mb, maxMB)
		mb = maxMB
	}
	return mb * bytesPerMB
}

// parseOperations returns the ordered list of maintenance operations to execute.
// Order follows Iceberg best practices: compact → rewrite_position_delete_files
// → expire_snapshots → remove_orphans → rewrite_manifests.
// Returns an error if any unknown operation is specified or the result would be empty.
func parseOperations(ops string) ([]string, error) {
	ops = strings.TrimSpace(strings.ToLower(ops))
	if ops == "" || ops == "all" {
		return []string{"compact", "rewrite_position_delete_files", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, nil
	}

	validOps := map[string]struct{}{
		"compact":                       {},
		"rewrite_position_delete_files": {},
		"expire_snapshots":              {},
		"remove_orphans":                {},
		"rewrite_manifests":             {},
	}

	requested := make(map[string]struct{})
	for _, op := range strings.Split(ops, ",") {
		op = strings.TrimSpace(op)
		if op == "" {
			continue
		}
		if _, ok := validOps[op]; !ok {
			return nil, fmt.Errorf("unknown maintenance operation %q (valid: compact, rewrite_position_delete_files, expire_snapshots, remove_orphans, rewrite_manifests)", op)
		}
		requested[op] = struct{}{}
	}

	// Return in canonical order: compact → rewrite_position_delete_files →
	// expire_snapshots → remove_orphans → rewrite_manifests
	canonicalOrder := []string{"compact", "rewrite_position_delete_files", "expire_snapshots", "remove_orphans", "rewrite_manifests"}
	var result []string
	for _, op := range canonicalOrder {
		if _, ok := requested[op]; ok {
			result = append(result, op)
		}
	}

	if len(result) == 0 {
		return nil, fmt.Errorf("no valid maintenance operations specified")
	}
	return result, nil
}

func extractMetadataVersion(metadataFileName string) int {
	// Parse "v3.metadata.json" or "v3-{nonce}.metadata.json" → 3
	name := strings.TrimPrefix(metadataFileName, "v")
	name = strings.TrimSuffix(name, ".metadata.json")
	// Strip any nonce suffix (e.g. "3-1709766000" → "3")
	if dashIdx := strings.Index(name, "-"); dashIdx > 0 {
		name = name[:dashIdx]
	}
	version, _ := strconv.Atoi(name)
	return version
}

// readStringConfig reads a string value from plugin config, with fallback.
func readStringConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback string) string {
	if values == nil {
		return fallback
	}
	value := values[field]
	if value == nil {
		return fallback
	}
	switch kind := value.Kind.(type) {
	case *plugin_pb.ConfigValue_StringValue:
		return kind.StringValue
	case *plugin_pb.ConfigValue_Int64Value:
		return strconv.FormatInt(kind.Int64Value, 10)
	case *plugin_pb.ConfigValue_DoubleValue:
		return strconv.FormatFloat(kind.DoubleValue, 'f', -1, 64)
	case *plugin_pb.ConfigValue_BoolValue:
		return strconv.FormatBool(kind.BoolValue)
	default:
		glog.V(1).Infof("readStringConfig: unexpected config value type %T for field %q, using fallback", value.Kind, field)
	}
	return fallback
}

// readBoolConfig reads a bool value from plugin config, with fallback.
func readBoolConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback bool) bool {
	if values == nil {
		return fallback
	}
	value := values[field]
	if value == nil {
		return fallback
	}
	switch kind := value.Kind.(type) {
	case *plugin_pb.ConfigValue_BoolValue:
		return kind.BoolValue
	case *plugin_pb.ConfigValue_StringValue:
		s := strings.TrimSpace(strings.ToLower(kind.StringValue))
		if s == "true" || s == "1" || s == "yes" {
			return true
		}
		if s == "false" || s == "0" || s == "no" {
			return false
		}
		glog.V(1).Infof("readBoolConfig: unrecognized string value %q for field %q, using fallback %v", kind.StringValue, field, fallback)
	case *plugin_pb.ConfigValue_Int64Value:
		return kind.Int64Value != 0
	default:
		glog.V(1).Infof("readBoolConfig: unexpected config value type %T for field %q, using fallback", value.Kind, field)
	}
	return fallback
}

// readInt64Config reads an int64 value from plugin config, with fallback.
func readInt64Config(values map[string]*plugin_pb.ConfigValue, field string, fallback int64) int64 {
	if values == nil {
		return fallback
	}
	value := values[field]
	if value == nil {
		return fallback
	}
	switch kind := value.Kind.(type) {
	case *plugin_pb.ConfigValue_Int64Value:
		return kind.Int64Value
	case *plugin_pb.ConfigValue_DoubleValue:
		return int64(kind.DoubleValue)
	case *plugin_pb.ConfigValue_StringValue:
		parsed, err := strconv.ParseInt(strings.TrimSpace(kind.StringValue), 10, 64)
		if err == nil {
			return parsed
		}
	case *plugin_pb.ConfigValue_BoolValue:
		if kind.BoolValue {
			return 1
		}
		return 0
	default:
		glog.V(1).Infof("readInt64Config: unexpected config value type %T for field %q, using fallback", value.Kind, field)
	}
	return fallback
}