mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-01 09:50:39 +01:00
Chris Lu
a80259d362
#10774 gave buildTestMetadata its refs and age parameters while #10773 added a caller with the old arity. Each was green against a master that did not yet have the other, and the merge of both does not compile, so vet and the unit tests fail on master.
407 lines
16 KiB
Go
407 lines
16 KiB
Go
package iceberg
|
|
|
|
import (
|
|
"errors"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/apache/iceberg-go"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
)
|
|
|
|
func settingPtr(v int64) *int64 { return &v }
|
|
|
|
func baseTestConfig() Config {
|
|
return applyThresholdDefaults(Config{
|
|
SnapshotRetentionMs: hoursToMs(defaultSnapshotRetentionHours),
|
|
MaxSnapshotsToKeep: defaultMaxSnapshotsToKeep,
|
|
TablePropertiesOverride: true,
|
|
})
|
|
}
|
|
|
|
func TestResolveTableConfigNoProperties(t *testing.T) {
|
|
base := baseTestConfig()
|
|
|
|
got := resolveTableConfig(base, iceberg.Properties{}, nil)
|
|
if got != base {
|
|
t.Errorf("expected config untouched, got %+v", got)
|
|
}
|
|
|
|
if got := resolveTableConfig(base, nil, nil); got != base {
|
|
t.Errorf("expected config untouched for nil properties, got %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveTableConfigPropertiesWin(t *testing.T) {
|
|
base := baseTestConfig()
|
|
got := resolveTableConfig(base, iceberg.Properties{
|
|
propTargetFileSize: "536870912",
|
|
propDeleteTargetFileSize: "33554432",
|
|
propMaxSnapshotAgeMs: "432000000",
|
|
propMinSnapshotsToKeep: "1",
|
|
}, nil)
|
|
|
|
if got.TargetFileSizeBytes != 536870912 {
|
|
t.Errorf("expected TargetFileSizeBytes=536870912, got %d", got.TargetFileSizeBytes)
|
|
}
|
|
if got.DeleteTargetFileSizeBytes != 33554432 {
|
|
t.Errorf("expected DeleteTargetFileSizeBytes=33554432, got %d", got.DeleteTargetFileSizeBytes)
|
|
}
|
|
if got.SnapshotRetentionMs != 432000000 {
|
|
t.Errorf("expected SnapshotRetentionMs=432000000, got %d", got.SnapshotRetentionMs)
|
|
}
|
|
if got.MaxSnapshotsToKeep != 1 {
|
|
t.Errorf("expected MaxSnapshotsToKeep=1, got %d", got.MaxSnapshotsToKeep)
|
|
}
|
|
}
|
|
|
|
// A sub-hour retention has to survive resolution; truncating it to whole hours
|
|
// would round down to zero and get clamped back up to the default.
|
|
func TestResolveTableConfigSubHourRetention(t *testing.T) {
|
|
got := resolveTableConfig(baseTestConfig(), iceberg.Properties{propMaxSnapshotAgeMs: "1"}, nil)
|
|
if got.SnapshotRetentionMs != 1 {
|
|
t.Errorf("expected SnapshotRetentionMs=1, got %d", got.SnapshotRetentionMs)
|
|
}
|
|
}
|
|
|
|
func TestResolveTableConfigIgnoresUnusableValues(t *testing.T) {
|
|
base := baseTestConfig()
|
|
|
|
for name, value := range map[string]string{
|
|
"not a number": "512mb",
|
|
"empty": "",
|
|
"zero": "0",
|
|
"negative": "-1",
|
|
"overflow": "99999999999999999999",
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
got := resolveTableConfig(base, iceberg.Properties{propTargetFileSize: value}, nil)
|
|
if got.TargetFileSizeBytes != base.TargetFileSizeBytes {
|
|
t.Errorf("expected fallback to %d, got %d", base.TargetFileSizeBytes, got.TargetFileSizeBytes)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResolveTableConfigTrimsWhitespace(t *testing.T) {
|
|
got := resolveTableConfig(baseTestConfig(), iceberg.Properties{propTargetFileSize: " 536870912\n"}, nil)
|
|
if got.TargetFileSizeBytes != 536870912 {
|
|
t.Errorf("expected TargetFileSizeBytes=536870912, got %d", got.TargetFileSizeBytes)
|
|
}
|
|
}
|
|
|
|
func TestResolveTableConfigLeavesOtherFields(t *testing.T) {
|
|
base := baseTestConfig()
|
|
base.Operations = "compact"
|
|
base.Where = "day = 3"
|
|
base.MinInputFiles = 9
|
|
|
|
got := resolveTableConfig(base, iceberg.Properties{propTargetFileSize: "536870912"}, nil)
|
|
if got.Operations != "compact" || got.Where != "day = 3" || got.MinInputFiles != 9 {
|
|
t.Errorf("expected unrelated fields preserved, got %+v", got)
|
|
}
|
|
}
|
|
|
|
// An orphan cutoff large enough to overflow time.Duration would wrap negative,
|
|
// put the cutoff in the future and make every file look like an orphan.
|
|
func TestApplyThresholdDefaultsClampsOrphanCutoff(t *testing.T) {
|
|
got := applyThresholdDefaults(Config{OrphanOlderThanHours: math.MaxInt64})
|
|
if got.OrphanOlderThanHours != maxOrphanOlderThanHours {
|
|
t.Fatalf("expected the cutoff clamped to %d, got %d", maxOrphanOlderThanHours, got.OrphanOlderThanHours)
|
|
}
|
|
if cutoff := time.Duration(got.OrphanOlderThanHours) * time.Hour; cutoff <= 0 {
|
|
t.Errorf("expected a positive cutoff duration, got %v", cutoff)
|
|
}
|
|
|
|
if got := applyThresholdDefaults(Config{OrphanOlderThanHours: 72}); got.OrphanOlderThanHours != 72 {
|
|
t.Errorf("expected a normal cutoff untouched, got %d", got.OrphanOlderThanHours)
|
|
}
|
|
}
|
|
|
|
func maintenanceConfig(entries map[string]*s3tables.MaintenanceConfigurationValue) s3tables.MaintenanceConfiguration {
|
|
return s3tables.MaintenanceConfiguration(entries)
|
|
}
|
|
|
|
func TestResolveTableConfigAppliesMaintenanceConfig(t *testing.T) {
|
|
got := resolveTableConfig(baseTestConfig(), nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{IcebergCompaction: &s3tables.IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
|
|
},
|
|
s3tables.MaintenanceTypeIcebergSnapshotManagement: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{IcebergSnapshotManagement: &s3tables.IcebergSnapshotManagementSettings{
|
|
MinSnapshotsToKeep: settingPtr(2),
|
|
MaxSnapshotAgeHours: settingPtr(120),
|
|
}},
|
|
},
|
|
}))
|
|
|
|
if got.TargetFileSizeBytes != 512*bytesPerMB {
|
|
t.Errorf("expected TargetFileSizeBytes=%d, got %d", 512*bytesPerMB, got.TargetFileSizeBytes)
|
|
}
|
|
if got.MaxSnapshotsToKeep != 2 {
|
|
t.Errorf("expected MaxSnapshotsToKeep=2, got %d", got.MaxSnapshotsToKeep)
|
|
}
|
|
if got.SnapshotRetentionMs != hoursToMs(120) {
|
|
t.Errorf("expected SnapshotRetentionMs=%d, got %d", hoursToMs(120), got.SnapshotRetentionMs)
|
|
}
|
|
}
|
|
|
|
// The default: a table declaring its own layout beats the control plane, so a
|
|
// writer and the compactor cannot disagree about target file size.
|
|
func TestResolveTableConfigPropertyBeatsMaintenanceConfig(t *testing.T) {
|
|
maintenance := maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{IcebergCompaction: &s3tables.IcebergCompactionSettings{TargetFileSizeMB: settingPtr(128)}},
|
|
},
|
|
})
|
|
props := iceberg.Properties{propTargetFileSize: "536870912"}
|
|
|
|
got := resolveTableConfig(baseTestConfig(), props, maintenance)
|
|
if got.TargetFileSizeBytes != 536870912 {
|
|
t.Errorf("expected the property to win with 536870912, got %d", got.TargetFileSizeBytes)
|
|
}
|
|
|
|
base := baseTestConfig()
|
|
base.TablePropertiesOverride = false
|
|
got = resolveTableConfig(base, props, maintenance)
|
|
if got.TargetFileSizeBytes != 128*bytesPerMB {
|
|
t.Errorf("expected the maintenance config to win with %d, got %d", 128*bytesPerMB, got.TargetFileSizeBytes)
|
|
}
|
|
}
|
|
|
|
// A disabled type drops its operations rather than contributing settings.
|
|
func TestResolveTableConfigIgnoresDisabledSettings(t *testing.T) {
|
|
base := baseTestConfig()
|
|
got := resolveTableConfig(base, nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {
|
|
Status: s3tables.MaintenanceStatusDisabled,
|
|
Settings: &s3tables.MaintenanceSettings{IcebergCompaction: &s3tables.IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
|
|
},
|
|
}))
|
|
|
|
if got.TargetFileSizeBytes != base.TargetFileSizeBytes {
|
|
t.Errorf("expected disabled settings ignored, got %d", got.TargetFileSizeBytes)
|
|
}
|
|
}
|
|
|
|
func TestFilterDisabledOperations(t *testing.T) {
|
|
all := []string{"compact", "rewrite_position_delete_files", "expire_snapshots", "remove_orphans", "rewrite_manifests"}
|
|
|
|
if got := filterDisabledOperations(all, nil); len(got) != len(all) {
|
|
t.Errorf("expected no filtering without a configuration, got %v", got)
|
|
}
|
|
|
|
got := filterDisabledOperations(all, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {Status: s3tables.MaintenanceStatusDisabled},
|
|
s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: s3tables.MaintenanceStatusDisabled},
|
|
}))
|
|
want := []string{"expire_snapshots"}
|
|
if len(got) != len(want) || got[0] != want[0] {
|
|
t.Errorf("expected %v, got %v", want, got)
|
|
}
|
|
|
|
if got := filterDisabledOperations(all, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {Status: s3tables.MaintenanceStatusEnabled},
|
|
})); len(got) != len(all) {
|
|
t.Errorf("expected an enabled type to filter nothing, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestMergeMaintenanceConfigurationTableWins(t *testing.T) {
|
|
bucket := maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: s3tables.MaintenanceStatusEnabled},
|
|
s3tables.MaintenanceTypeIcebergCompaction: {Status: s3tables.MaintenanceStatusEnabled},
|
|
})
|
|
table := maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {Status: s3tables.MaintenanceStatusDisabled},
|
|
})
|
|
|
|
merged := mergeMaintenanceConfiguration(bucket, table)
|
|
if merged[s3tables.MaintenanceTypeIcebergCompaction].Status != s3tables.MaintenanceStatusDisabled {
|
|
t.Error("expected the table entry to win")
|
|
}
|
|
if merged[s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval].Status != s3tables.MaintenanceStatusEnabled {
|
|
t.Error("expected the bucket-only entry retained")
|
|
}
|
|
|
|
if got := mergeMaintenanceConfiguration(nil, table); len(got) != 1 {
|
|
t.Errorf("expected the table configuration passed through, got %v", got)
|
|
}
|
|
if got := mergeMaintenanceConfiguration(bucket, nil); len(got) != 2 {
|
|
t.Errorf("expected the bucket configuration passed through, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestParseMaintenanceConfiguration(t *testing.T) {
|
|
got, err := parseMaintenanceConfiguration(nil, "bucket")
|
|
if err != nil || got != nil {
|
|
t.Errorf("expected nil for a missing attribute, got %v (%v)", got, err)
|
|
}
|
|
|
|
// Unreadable must be an error, not an empty configuration: empty would run
|
|
// operations the operator disabled.
|
|
if _, err := parseMaintenanceConfiguration(map[string][]byte{s3tables.ExtendedKeyMaintenance: []byte("{")}, "bucket"); err == nil {
|
|
t.Error("expected an error for a malformed attribute")
|
|
}
|
|
|
|
data := []byte(`{"icebergCompaction":{"status":"disabled"}}`)
|
|
got, err = parseMaintenanceConfiguration(map[string][]byte{s3tables.ExtendedKeyMaintenance: data}, "bucket")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got[s3tables.MaintenanceTypeIcebergCompaction].Status != s3tables.MaintenanceStatusDisabled {
|
|
t.Errorf("expected compaction disabled, got %v", got)
|
|
}
|
|
}
|
|
|
|
// unreferencedDays large enough to overflow must not land a cutoff in the future.
|
|
func TestResolveTableConfigClampsUnreferencedDays(t *testing.T) {
|
|
got := resolveTableConfig(baseTestConfig(), nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{
|
|
IcebergUnreferencedFileRemoval: &s3tables.IcebergUnreferencedFileRemovalSettings{UnreferencedDays: settingPtr(math.MaxInt64)},
|
|
},
|
|
},
|
|
}))
|
|
|
|
if got.OrphanOlderThanHours != maxOrphanOlderThanHours {
|
|
t.Fatalf("expected the cutoff clamped to %d, got %d", maxOrphanOlderThanHours, got.OrphanOlderThanHours)
|
|
}
|
|
if cutoff := time.Duration(got.OrphanOlderThanHours) * time.Hour; cutoff <= 0 {
|
|
t.Errorf("expected a positive cutoff duration, got %v", cutoff)
|
|
}
|
|
|
|
got = resolveTableConfig(baseTestConfig(), nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{
|
|
IcebergUnreferencedFileRemoval: &s3tables.IcebergUnreferencedFileRemovalSettings{UnreferencedDays: settingPtr(3)},
|
|
},
|
|
},
|
|
}))
|
|
if got.OrphanOlderThanHours != 72 {
|
|
t.Errorf("expected 3 days to be 72 hours, got %d", got.OrphanOlderThanHours)
|
|
}
|
|
}
|
|
|
|
// AWS marks a file non-current after unreferencedDays and deletes it
|
|
// nonCurrentDays later; deleting in one step has to wait for both.
|
|
func TestResolveTableConfigAddsNonCurrentWindow(t *testing.T) {
|
|
orphan := func(unreferenced, nonCurrent int64) Config {
|
|
return resolveTableConfig(baseTestConfig(), nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{
|
|
IcebergUnreferencedFileRemoval: &s3tables.IcebergUnreferencedFileRemovalSettings{
|
|
UnreferencedDays: settingPtr(unreferenced),
|
|
NonCurrentDays: settingPtr(nonCurrent),
|
|
},
|
|
},
|
|
},
|
|
}))
|
|
}
|
|
|
|
if got := orphan(3, 10); got.OrphanOlderThanHours != 13*24 {
|
|
t.Errorf("expected 3+10 days to be %d hours, got %d", 13*24, got.OrphanOlderThanHours)
|
|
}
|
|
if got := orphan(0, 10); got.OrphanOlderThanHours != 10*24 {
|
|
t.Errorf("expected nonCurrentDays alone to apply, got %d", got.OrphanOlderThanHours)
|
|
}
|
|
if got := orphan(math.MaxInt64, math.MaxInt64); got.OrphanOlderThanHours != maxOrphanOlderThanHours {
|
|
t.Errorf("expected the sum clamped, got %d", got.OrphanOlderThanHours)
|
|
}
|
|
}
|
|
|
|
func TestBuildJobStatus(t *testing.T) {
|
|
ranAt := time.Unix(1755250000, 0).UTC()
|
|
|
|
jobStatus := buildJobStatus(map[string]error{
|
|
"compact": nil,
|
|
"expire_snapshots": nil,
|
|
}, ranAt)
|
|
|
|
compaction := jobStatus[s3tables.MaintenanceTypeIcebergCompaction]
|
|
if compaction == nil || compaction.Status != s3tables.MaintenanceJobStatusSuccessful {
|
|
t.Fatalf("expected compaction successful, got %+v", compaction)
|
|
}
|
|
if compaction.LastRunTimestamp == nil || !compaction.LastRunTimestamp.Equal(ranAt) {
|
|
t.Errorf("expected lastRunTimestamp %v, got %v", ranAt, compaction.LastRunTimestamp)
|
|
}
|
|
if _, ok := jobStatus[s3tables.MaintenanceTypeIcebergUnreferencedFileRemoval]; ok {
|
|
t.Error("expected an operation that did not run to be left out")
|
|
}
|
|
}
|
|
|
|
// Several operations share one AWS job type, so one failure fails the type.
|
|
func TestBuildJobStatusFailureWins(t *testing.T) {
|
|
jobStatus := buildJobStatus(map[string]error{
|
|
"compact": nil,
|
|
"rewrite_manifests": errors.New("commit conflict"),
|
|
}, time.Unix(1755250000, 0).UTC())
|
|
|
|
compaction := jobStatus[s3tables.MaintenanceTypeIcebergCompaction]
|
|
if compaction.Status != s3tables.MaintenanceJobStatusFailed {
|
|
t.Errorf("expected compaction failed, got %q", compaction.Status)
|
|
}
|
|
if compaction.FailureMessage != "commit conflict" {
|
|
t.Errorf("expected the failure message carried, got %q", compaction.FailureMessage)
|
|
}
|
|
}
|
|
|
|
func TestBuildJobStatusEmpty(t *testing.T) {
|
|
if got := buildJobStatus(nil, time.Now()); len(got) != 0 {
|
|
t.Errorf("expected an empty status, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveTableConfigAppliesCompactionStrategy(t *testing.T) {
|
|
withStrategy := func(strategy string) Config {
|
|
return resolveTableConfig(baseTestConfig(), nil, maintenanceConfig(map[string]*s3tables.MaintenanceConfigurationValue{
|
|
s3tables.MaintenanceTypeIcebergCompaction: {
|
|
Status: s3tables.MaintenanceStatusEnabled,
|
|
Settings: &s3tables.MaintenanceSettings{IcebergCompaction: &s3tables.IcebergCompactionSettings{Strategy: strategy}},
|
|
},
|
|
}))
|
|
}
|
|
|
|
if got := withStrategy(s3tables.CompactionStrategySort); got.RewriteStrategy != "sort" {
|
|
t.Errorf("expected sort, got %q", got.RewriteStrategy)
|
|
}
|
|
if got := withStrategy(s3tables.CompactionStrategyBinpack); got.RewriteStrategy != "binpack" {
|
|
t.Errorf("expected binpack, got %q", got.RewriteStrategy)
|
|
}
|
|
|
|
// "auto" is carried through so the plan can pick per table.
|
|
if got := withStrategy(s3tables.CompactionStrategyAuto); got.RewriteStrategy != rewriteStrategyAuto {
|
|
t.Errorf("expected auto preserved, got %q", got.RewriteStrategy)
|
|
}
|
|
}
|
|
|
|
// AWS defines auto as sorting tables that declare a sort order and bin-packing
|
|
// the rest, so it must not collapse to the worker default.
|
|
func TestResolveCompactionRewritePlanAuto(t *testing.T) {
|
|
cfg := baseTestConfig()
|
|
cfg.RewriteStrategy = rewriteStrategyAuto
|
|
|
|
unsorted := buildTestMetadata(t, nil, nil, 0)
|
|
plan, err := resolveCompactionRewritePlan(cfg, unsorted)
|
|
if err != nil {
|
|
t.Fatalf("auto must not fail on an unsorted table: %v", err)
|
|
}
|
|
if plan.strategy != defaultRewriteStrategy {
|
|
t.Errorf("expected binpack for an unsorted table, got %q", plan.strategy)
|
|
}
|
|
|
|
// An explicit sort request on the same table is still an error, so auto is
|
|
// doing something the existing strategies did not.
|
|
cfg.RewriteStrategy = "sort"
|
|
if _, err := resolveCompactionRewritePlan(cfg, unsorted); err == nil {
|
|
t.Error("expected explicit sort to fail without a table sort order")
|
|
}
|
|
}
|