seaweedfs/weed/worker/tasks/iceberg/table_config.go

package iceberg

import (
	"context"
	"encoding/json"
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/apache/iceberg-go"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)

// Iceberg table properties that describe the physical layout maintenance has
// to produce.
const (
	propTargetFileSize       = "write.target-file-size-bytes"
	propDeleteTargetFileSize = "write.delete.target-file-size-bytes"
	propRowGroupSize         = "write.parquet.row-group-size-bytes"
	propRowGroupRowLimit     = "write.parquet.row-group-limit"
	propMaxSnapshotAgeMs     = "history.expire.max-snapshot-age-ms"
	propMinSnapshotsToKeep   = "history.expire.min-snapshots-to-keep"
)

// resolveTableConfig layers a table's own intent over the worker config, from
// the maintenance configuration and from the table's Iceberg properties.
//
// Properties win by default: a writer targeting 512 MiB and a compactor
// rewriting to 256 MiB would rewrite each other's output forever, and the
// property is the signal every engine already honours. Operators who want the
// control plane to be authoritative instead can clear
// table_properties_override.
func resolveTableConfig(base Config, props iceberg.Properties, maintenance s3tables.MaintenanceConfiguration) Config {
	cfg := base
	if base.TablePropertiesOverride {
		cfg = applyMaintenanceConfig(cfg, maintenance)
		cfg = applyTableProperties(cfg, props)
	} else {
		cfg = applyTableProperties(cfg, props)
		cfg = applyMaintenanceConfig(cfg, maintenance)
	}
	return applyThresholdDefaults(cfg)
}

func applyTableProperties(cfg Config, props iceberg.Properties) Config {
	if v, ok := propInt64(props, propTargetFileSize); ok {
		cfg.TargetFileSizeBytes = v
	}
	if v, ok := propInt64(props, propDeleteTargetFileSize); ok {
		cfg.DeleteTargetFileSizeBytes = v
	}
	if v, ok := propInt64(props, propRowGroupSize); ok {
		cfg.RowGroupSizeBytes = v
	}
	if v, ok := propInt64(props, propRowGroupRowLimit); ok {
		cfg.RowGroupRowLimit = v
	}
	if v, ok := propInt64(props, propMaxSnapshotAgeMs); ok {
		cfg.SnapshotRetentionMs = v
	}
	if v, ok := propInt64(props, propMinSnapshotsToKeep); ok {
		cfg.MaxSnapshotsToKeep = v
	}
	return cfg
}

func applyMaintenanceConfig(cfg Config, maintenance s3tables.MaintenanceConfiguration) Config {
	if s := compactionSettings(maintenance); s != nil {
		if v := settingValue(s.TargetFileSizeMB); v > 0 {
			cfg.TargetFileSizeBytes = mbToBytes(v)
		}
		switch s.Strategy {
		case s3tables.CompactionStrategyBinpack, s3tables.CompactionStrategySort, s3tables.CompactionStrategyAuto:
			cfg.RewriteStrategy = s.Strategy
		}
	}
	if s := snapshotSettings(maintenance); s != nil {
		if v := settingValue(s.MaxSnapshotAgeHours); v > 0 {
			cfg.SnapshotRetentionMs = hoursToMs(v)
		}
		if v := settingValue(s.MinSnapshotsToKeep); v > 0 {
			cfg.MaxSnapshotsToKeep = v
		}
	}
	// AWS marks a file non-current after unreferencedDays and deletes it a
	// further nonCurrentDays later. remove_orphans deletes in one step, so the
	// cutoff has to be the sum or the recovery window disappears.
	if s := orphanSettings(maintenance); s != nil {
		if days := addDays(settingValue(s.UnreferencedDays), settingValue(s.NonCurrentDays)); days > 0 {
			cfg.OrphanOlderThanHours = daysToHours(days)
		}
	}
	return cfg
}

// settingValue reads an optional maintenance setting, treating unset and
// nonsensical values alike so the worker keeps its own configuration.
func settingValue(v *int64) int64 {
	if v == nil || *v < 0 {
		return 0
	}
	return *v
}

func compactionSettings(maintenance s3tables.MaintenanceConfiguration) *s3tables.IcebergCompactionSettings {
	if v := enabledSettings(maintenance, s3tables.MaintenanceTypeIcebergCompaction); v != nil {
		return v.IcebergCompaction
	}
	return nil
}

func snapshotSettings(maintenance s3tables.MaintenanceConfiguration) *s3tables.IcebergSnapshotManagementSettings {
	if v := enabledSettings(maintenance, s3tables.MaintenanceTypeIcebergSnapshotManagement); v != nil {
		return v.IcebergSnapshotManagement
	}
	return nil
}

func orphanSettings(maintenance s3tables.MaintenanceConfiguration) *s3tables.IcebergUnreferencedFileRemovalSettings {
	if v := enabledSettings(maintenance, s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval); v != nil {
		return v.IcebergUnreferencedFileRemoval
	}
	return nil
}

// enabledSettings returns the settings for a maintenance type only when it is
// switched on; a disabled type drops its operations instead of retuning them.
func enabledSettings(maintenance s3tables.MaintenanceConfiguration, maintenanceType string) *s3tables.MaintenanceSettings {
	value, ok := maintenance[maintenanceType]
	if !ok || value == nil || value.Status == s3tables.MaintenanceStatusDisabled {
		return nil
	}
	return value.Settings
}

var mergeMaintenanceConfiguration = s3tables.MergeMaintenanceConfiguration

// maintenanceJobType maps a worker operation onto the AWS job type that governs
// it. Manifest and delete-file rewrites have no AWS equivalent and ride along
// with compaction, so disabling compaction stops all rewriting.
func maintenanceJobType(op string) string {
	switch op {
	case "compact", "rewrite_position_delete_files", "rewrite_manifests":
		return s3tables.MaintenanceTypeIcebergCompaction
	case "expire_snapshots":
		return s3tables.MaintenanceTypeIcebergSnapshotManagement
	case "remove_orphans":
		return s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval
	}
	return ""
}

// filterDisabledOperations drops the operations the maintenance configuration
// switched off. No table property can re-enable them.
func filterDisabledOperations(ops []string, maintenance s3tables.MaintenanceConfiguration) []string {
	if len(maintenance) == 0 {
		return ops
	}

	filtered := make([]string, 0, len(ops))
	for _, op := range ops {
		value, ok := maintenance[maintenanceJobType(op)]
		if ok && value != nil && value.Status == s3tables.MaintenanceStatusDisabled {
			continue
		}
		filtered = append(filtered, op)
	}
	return filtered
}

// loadMaintenanceConfiguration reads the bucket and table maintenance
// configuration and overlays the table's on the bucket's. Detection gets both
// from entries it already listed; execution has to look them up.
func loadMaintenanceConfiguration(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, tablePath string) (s3tables.MaintenanceConfiguration, error) {
	bucketDir := path.Join(s3tables.TablesPath, bucketName)
	tableDir := path.Join(bucketDir, tablePath)

	bucket, err := lookupMaintenanceConfiguration(ctx, client, bucketDir, bucketName)
	if err != nil {
		return nil, err
	}
	table, err := lookupMaintenanceConfiguration(ctx, client, tableDir, path.Join(bucketName, tablePath))
	if err != nil {
		return nil, err
	}
	return mergeMaintenanceConfiguration(bucket, table), nil
}

func lookupMaintenanceConfiguration(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, resource string) (s3tables.MaintenanceConfiguration, error) {
	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
		Directory: path.Dir(dir),
		Name:      path.Base(dir),
	})
	if err != nil {
		return nil, fmt.Errorf("look up %s: %w", resource, err)
	}
	if resp == nil || resp.Entry == nil {
		return nil, nil
	}
	return parseMaintenanceConfiguration(resp.Entry.Extended, resource)
}

// parseMaintenanceConfiguration reads the maintenance configuration stored on a
// filer entry. An unreadable attribute is an error rather than an empty
// configuration: treating it as empty would run operations the operator
// disabled.
func parseMaintenanceConfiguration(extended map[string][]byte, resource string) (s3tables.MaintenanceConfiguration, error) {
	data, ok := extended[s3tables.ExtendedKeyMaintenance]
	if !ok || len(data) == 0 {
		return nil, nil
	}

	var config s3tables.MaintenanceConfiguration
	if err := json.Unmarshal(data, &config); err != nil {
		return nil, fmt.Errorf("parse maintenance configuration on %s: %w", resource, err)
	}
	return config, nil
}

// propInt64 reads a positive integer table property. Anything unset,
// unparseable or non-positive leaves the worker config in place so a
// misconfigured table never stops its own maintenance.
func propInt64(props iceberg.Properties, key string) (int64, bool) {
	raw, ok := props[key]
	if !ok {
		return 0, false
	}
	value, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64)
	if err != nil {
		glog.V(1).Infof("iceberg maintenance: ignoring table property %s=%q: not an integer", key, raw)
		return 0, false
	}
	if value <= 0 {
		glog.V(1).Infof("iceberg maintenance: ignoring table property %s=%d: not positive", key, value)
		return 0, false
	}
	return value, true
}