seaweedfs/weed/s3api/s3tables/iceberg_manifest_test.go

package s3tables

import (
	"bytes"
	"testing"
	"time"

	"github.com/apache/iceberg-go"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
	"github.com/twmb/avro/ocf"
)

// A manifest whose partition union puts the value branch first hides the
// logical type from iceberg-go, which then hands back whatever the Avro decoder
// produced even once the partition is read. Rewriting such an entry fails when
// the writer builds its partition summaries, or encodes the wrong number for a
// time partition, so every logical type a partition field can carry has to come
// back as an Iceberg value.
func TestReadManifestNormalizesForeignPartitions(t *testing.T) {
	day := time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC)

	cases := []struct {
		name        string
		sourceType  iceberg.PrimitiveType
		transform   iceberg.Transform
		logicalType string
		// decoded is the value the foreign manifest carries, as the Avro
		// decoder hands it back.
		decoded any
		want    any
		// nullFirst spells the union the way Java and iceberg-rust do, which
		// iceberg-go converts on its own once the partition is read.
		nullFirst bool
		// rawIsNormalized marks transforms iceberg-go now converts itself on
		// read (day transforms), so the raw value already matches want.
		rawIsNormalized bool
	}{
		{
			name:        "day transform written null-first converts on read",
			sourceType:  iceberg.PrimitiveTypes.Timestamp,
			transform:   iceberg.DayTransform{},
			logicalType: "date",
			decoded:     day,
			want:        iceberg.Date(20737),
			nullFirst:   true,
		},
		{
			name:        "day transform on a timestamp column",
			sourceType:  iceberg.PrimitiveTypes.Timestamp,
			transform:   iceberg.DayTransform{},
			logicalType: "date",
			decoded:     day,
			want:        iceberg.Date(20737),
			// iceberg-go now converts day-transform partitions to iceberg.Date
			// itself (applyDayTransformDates), so the raw read already yields
			// the Iceberg value regardless of union branch ordering.
			rawIsNormalized: true,
		},
		{
			name:        "identity transform on a date column",
			sourceType:  iceberg.PrimitiveTypes.Date,
			transform:   iceberg.IdentityTransform{},
			logicalType: "date",
			decoded:     day,
			want:        iceberg.Date(20737),
		},
		{
			name:        "identity transform on a timestamp column",
			sourceType:  iceberg.PrimitiveTypes.Timestamp,
			transform:   iceberg.IdentityTransform{},
			logicalType: "timestamp-micros",
			decoded:     day.Add(3 * time.Hour),
			want:        iceberg.Timestamp(day.Add(3 * time.Hour).UnixMicro()),
		},
		{
			name:        "identity transform on a time column",
			sourceType:  iceberg.PrimitiveTypes.Time,
			transform:   iceberg.IdentityTransform{},
			logicalType: "time-micros",
			decoded:     3 * time.Hour,
			want:        iceberg.Time((3 * time.Hour).Microseconds()),
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			schema := iceberg.NewSchema(0,
				iceberg.NestedField{ID: 1, Name: "source", Type: c.sourceType, Required: true},
			)
			spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
				SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: c.transform,
			})
			specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}

			dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
				iceberg.ParquetFile, map[int]any{1000: c.want}, nil, nil, 1, 1)
			if err != nil {
				t.Fatalf("build data file: %v", err)
			}
			snapshotID := int64(1)
			entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())

			foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
				"metadata/foreign-manifest.avro", c.logicalType, c.decoded, !c.nullFirst)

			// iceberg-go reads the logical type off the union's last branch, so
			// the ordering decides whether the entry arrives as an Iceberg value
			// or as whatever the Avro decoder produced.
			rawWant := any(c.decoded)
			if c.nullFirst || c.rawIsNormalized {
				rawWant = c.want
			}
			raw, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
			if err != nil {
				t.Fatalf("read foreign manifest: %v", err)
			}
			if got := raw[0].DataFile().Partition()[1000]; got != rawWant {
				t.Fatalf("unnormalized partition value = %#v (%T), want %#v (%T)", got, got, rawWant, rawWant)
			}

			entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
			if err != nil {
				t.Fatalf("read foreign manifest: %v", err)
			}
			if got := entries[0].DataFile().Partition()[1000]; got != c.want {
				t.Fatalf("normalized partition value = %#v (%T), want %#v (%T)", got, got, c.want, c.want)
			}

			// The point of normalizing: the entry can be written out again.
			var buf bytes.Buffer
			if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
				t.Fatalf("write normalized manifest: %v", err)
			}
		})
	}
}

// A manifest written under a spec the table metadata no longer records is
// returned as read rather than refused: the caller decides what to do with it.
func TestReadManifestUnknownSpec(t *testing.T) {
	schema := iceberg.NewSchema(0,
		iceberg.NestedField{ID: 1, Name: "source", Type: iceberg.PrimitiveTypes.Date, Required: true},
	)
	spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
		SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
	})
	dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
		iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
	if err != nil {
		t.Fatalf("build data file: %v", err)
	}
	snapshotID := int64(1)
	entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
	foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
		"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)

	entries, err := ReadManifest(foreignManifest, foreignBytes, true, nil, schema)
	if err != nil {
		t.Fatalf("read foreign manifest: %v", err)
	}
	if _, ok := entries[0].DataFile().Partition()[1000].(time.Time); !ok {
		t.Fatalf("partition value = %T, want it returned as read", entries[0].DataFile().Partition()[1000])
	}
}

// iceberg-go now converts day-transform partitions to iceberg.Date during read
// (applyDayTransformDates), so an entry nobody looked at between reading and
// writing still carries an Iceberg value and writes cleanly. This used to fail
// because conversion was lazy and ManifestWriter.addEntry rebound the logical
// types to the output manifest — whose day transform carries no date logical
// type — before the first Partition() call, leaving a time.Time that the
// encoder rejected. The shim keeps the round-trip working for the identity
// transforms iceberg-go still does not convert (see TestReadManifestNormalizesForeignPartitions).
func TestReadManifestConvertsBeforeTheWriterRebindsLogicalTypes(t *testing.T) {
	schema := iceberg.NewSchema(0,
		iceberg.NestedField{ID: 1, Name: "event_time", Type: iceberg.PrimitiveTypes.Timestamp, Required: true},
	)
	spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
		SourceIDs: []int{1}, FieldID: 1000, Name: "event_time_day", Transform: iceberg.DayTransform{},
	})
	specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
	dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
		iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
	if err != nil {
		t.Fatalf("build data file: %v", err)
	}
	snapshotID := int64(1)
	entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
	foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
		"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), false)

	untouched, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
	if err != nil {
		t.Fatalf("read foreign manifest: %v", err)
	}
	got := untouched[0].DataFile().Partition()[1000]
	if _, ok := got.(iceberg.Date); !ok {
		t.Fatalf("iceberg-go now converts day partitions on read; got %T, want iceberg.Date", got)
	}
	var buf bytes.Buffer
	if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, untouched); err != nil {
		t.Fatalf("write manifest read directly through iceberg-go: %v", err)
	}

	entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
	if err != nil {
		t.Fatalf("read foreign manifest: %v", err)
	}
	buf.Reset()
	if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
		t.Fatalf("write manifest read through the shim: %v", err)
	}
}

// rebuildManifestEntry rebuilds the DataFile through NewDataFileBuilder, which
// has no setter for the original block_size_in_bytes (the field is not exposed
// on DataFile). The v1 manifest schema includes block_size_in_bytes with a
// default of 64 MiB and the spec says "Always write default in v1", so a
// rebuilt entry written as v1 must carry that default — not 0.
func TestReadManifestRebuiltEntryPreservesV1BlockSizeDefault(t *testing.T) {
	schema := iceberg.NewSchema(0,
		iceberg.NestedField{ID: 1, Name: "event_date", Type: iceberg.PrimitiveTypes.Date, Required: true},
	)
	spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
		SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
	})
	specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
	dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
		iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
	if err != nil {
		t.Fatalf("build data file: %v", err)
	}
	snapshotID := int64(1)
	entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
	// valueFirst=true spells the union [<type>, null], hiding the logical type
	// from iceberg-go so the partition arrives as a time.Time. That triggers
	// normalizePartitionValue → rebuildManifestEntry, which is the path that
	// sets block_size_in_bytes.
	foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
		"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)

	entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
	if err != nil {
		t.Fatalf("read foreign manifest: %v", err)
	}

	var buf bytes.Buffer
	if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 1, spec, schema, snapshotID, entries); err != nil {
		t.Fatalf("write v1 manifest: %v", err)
	}

	reader, err := ocf.NewReader(bytes.NewReader(buf.Bytes()))
	if err != nil {
		t.Fatalf("open v1 manifest: %v", err)
	}
	var record map[string]any
	if err := reader.Decode(&record); err != nil {
		t.Fatalf("decode v1 manifest: %v", err)
	}
	if err := reader.Close(); err != nil {
		t.Fatalf("close v1 manifest: %v", err)
	}
	dataFile, ok := record["data_file"].(map[string]any)
	if !ok {
		t.Fatalf("data_file = %T, want record", record["data_file"])
	}
	got, ok := dataFile["block_size_in_bytes"]
	if !ok {
		t.Fatal("block_size_in_bytes missing from v1 manifest")
	}
	const want = int64(64 * 1024 * 1024)
	if got != want {
		t.Fatalf("block_size_in_bytes = %v, want %d", got, want)
	}
}