seaweedfs/weed/s3api/iceberg/iceberg_transaction_test.go

package iceberg

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"path"
	"strings"
	"testing"

	"github.com/apache/iceberg-go"
	"github.com/apache/iceberg-go/table"
	"github.com/google/uuid"
	"github.com/gorilla/mux"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
	"google.golang.org/grpc"
)

func TestDecodeCommitTransactionRequest(t *testing.T) {
	body := `{"table-changes":[
		{"identifier":{"namespace":["ns"],"name":"t1"},"requirements":[{"type":"assert-table-uuid","uuid":"00000000-0000-0000-0000-000000000001"}],"updates":[{"action":"set-properties","updates":{"k":"v"}}]},
		{"identifier":{"namespace":["ns","sub"],"name":"t2"},"requirements":[],"updates":[]}
	]}`

	var req CommitTransactionRequest
	if err := json.Unmarshal([]byte(body), &req); err != nil {
		t.Fatalf("Unmarshal() error = %v", err)
	}
	if len(req.TableChanges) != 2 {
		t.Fatalf("table-changes = %d, want 2", len(req.TableChanges))
	}
	if req.TableChanges[0].Identifier.Name != "t1" {
		t.Fatalf("change[0] name = %q, want t1", req.TableChanges[0].Identifier.Name)
	}
	if got := []string(req.TableChanges[1].Identifier.Namespace); len(got) != 2 || got[1] != "sub" {
		t.Fatalf("change[1] namespace = %v, want [ns sub]", got)
	}
}

func TestParseTableChangeSeparatesStatistics(t *testing.T) {
	change := tableChangeRequest{
		Requirements: json.RawMessage(`[{"type":"assert-table-uuid","uuid":"00000000-0000-0000-0000-000000000001"}]`),
		Updates: []json.RawMessage{
			json.RawMessage(`{"action":"set-properties","updates":{"k":"v"}}`),
			json.RawMessage(`{"action":"set-statistics","snapshot-id":10,"statistics-path":"s3://bucket/table/metadata/stats.puffin","file-size-in-bytes":100,"file-footer-size-in-bytes":20,"blob-metadata":[]}`),
		},
	}

	requirements, updates, statistics, reqErr := parseTableChange(change)
	if reqErr != nil {
		t.Fatalf("parseTableChange() error = %+v", reqErr)
	}
	if len(requirements) != 1 {
		t.Fatalf("requirements = %d, want 1", len(requirements))
	}
	if len(updates) != 1 {
		t.Fatalf("updates = %d, want 1", len(updates))
	}
	if len(statistics) != 1 || statistics[0].set == nil || statistics[0].set.SnapshotID != 10 {
		t.Fatalf("unexpected statistics updates: %#v", statistics)
	}
}

func TestParseTableChangeRejectsInvalidRequirements(t *testing.T) {
	change := tableChangeRequest{Requirements: json.RawMessage(`{not json}`)}
	_, _, _, reqErr := parseTableChange(change)
	if reqErr == nil {
		t.Fatalf("parseTableChange() expected error for invalid requirements")
	}
	if reqErr.status != 400 {
		t.Fatalf("status = %d, want 400", reqErr.status)
	}
}

func TestCloneTableMetadataIsIndependent(t *testing.T) {
	orig := &s3tables.TableMetadata{
		Iceberg:      &s3tables.IcebergMetadata{TableUUID: "uuid-1"},
		FullMetadata: json.RawMessage(`{"v":1}`),
	}
	clone := cloneTableMetadata(orig)
	if clone == orig || clone.Iceberg == orig.Iceberg {
		t.Fatalf("cloneTableMetadata returned aliased pointers")
	}

	orig.Iceberg.TableUUID = "mutated"
	orig.FullMetadata[0] = 'X'
	if clone.Iceberg.TableUUID != "uuid-1" {
		t.Fatalf("clone Iceberg.TableUUID = %q, want uuid-1", clone.Iceberg.TableUUID)
	}
	if string(clone.FullMetadata) != `{"v":1}` {
		t.Fatalf("clone FullMetadata = %q, mutated with source", clone.FullMetadata)
	}

	if cloneTableMetadata(nil) != nil {
		t.Fatalf("cloneTableMetadata(nil) = non-nil")
	}
}

func TestBuildTableRestoreRequestCarriesPriorState(t *testing.T) {
	pc := &preparedTableCommit{
		namespace:           []string{"ns", "sub"},
		tableName:           "t1",
		prevMetadataLoc:     "s3://bucket/ns/t1/metadata/v3.metadata.json",
		prevMetadataVersion: 3,
		prevMetadata: &s3tables.TableMetadata{
			Iceberg:      &s3tables.IcebergMetadata{TableUUID: uuid.NewString()},
			FullMetadata: json.RawMessage(`{"format-version":2,"last-updated-ms":3}`),
		},
	}

	req := buildTableRestoreRequest("arn:bucket", pc)

	if req.TableBucketARN != "arn:bucket" || req.Name != "t1" {
		t.Fatalf("restore target = %s/%s, want arn:bucket/t1", req.TableBucketARN, req.Name)
	}
	if req.MetadataLocation != pc.prevMetadataLoc {
		t.Fatalf("restore MetadataLocation = %q, want %q", req.MetadataLocation, pc.prevMetadataLoc)
	}
	if req.MetadataVersion != pc.prevMetadataVersion {
		t.Fatalf("restore MetadataVersion = %d, want %d", req.MetadataVersion, pc.prevMetadataVersion)
	}
	if req.Metadata == nil || !bytes.Equal(req.Metadata.FullMetadata, pc.prevMetadata.FullMetadata) {
		t.Fatalf("restore FullMetadata = %q, want %q", req.Metadata.FullMetadata, pc.prevMetadata.FullMetadata)
	}
	if req.Metadata.Iceberg == nil || req.Metadata.Iceberg.TableUUID != pc.prevMetadata.Iceberg.TableUUID {
		t.Fatalf("restore TableUUID mismatch")
	}
}

// TestRollbackRevertsEveryFlippedField mirrors handleUpdateTable's partial-field
// update semantics to prove that after a flip + rollback the table is back to its
// pre-transaction state. With the old location-only rollback the table kept the
// new full metadata and version; the restore request must revert all of them.
func TestRollbackRevertsEveryFlippedField(t *testing.T) {
	// prior on-disk state of an already-committed table.
	prior := tableEntry{
		metadataLocation: "s3://bucket/ns/t1/metadata/v3.metadata.json",
		metadataVersion:  3,
		metadata: &s3tables.TableMetadata{
			Iceberg:      &s3tables.IcebergMetadata{TableUUID: "uuid-1"},
			FullMetadata: json.RawMessage(`{"format-version":2,"v":3}`),
		},
	}

	pc := &preparedTableCommit{
		namespace:           []string{"ns"},
		tableName:           "t1",
		tableUUID:           uuid.MustParse("00000000-0000-0000-0000-000000000001"),
		metadataBytes:       json.RawMessage(`{"format-version":2,"v":4}`),
		metadataVersion:     4,
		newMetadataLoc:      "s3://bucket/ns/t1/metadata/v4.metadata.json",
		prevMetadataLoc:     prior.metadataLocation,
		prevMetadataVersion: prior.metadataVersion,
		prevMetadata:        cloneTableMetadata(prior.metadata),
	}

	// flip: apply the new commit, then roll it back via the restore request.
	state := prior.clone()
	state.applyUpdate(&s3tables.UpdateTableRequest{
		Metadata:         &s3tables.TableMetadata{Iceberg: &s3tables.IcebergMetadata{TableUUID: pc.tableUUID.String()}, FullMetadata: pc.metadataBytes},
		MetadataVersion:  pc.metadataVersion,
		MetadataLocation: pc.newMetadataLoc,
	})
	state.applyUpdate(buildTableRestoreRequest("arn:bucket", pc))

	if state.metadataLocation != prior.metadataLocation {
		t.Fatalf("after rollback location = %q, want %q", state.metadataLocation, prior.metadataLocation)
	}
	if state.metadataVersion != prior.metadataVersion {
		t.Fatalf("after rollback version = %d, want %d", state.metadataVersion, prior.metadataVersion)
	}
	if !bytes.Equal(state.metadata.FullMetadata, prior.metadata.FullMetadata) {
		t.Fatalf("after rollback FullMetadata = %q, want %q", state.metadata.FullMetadata, prior.metadata.FullMetadata)
	}
	if state.metadata.Iceberg.TableUUID != prior.metadata.Iceberg.TableUUID {
		t.Fatalf("after rollback TableUUID = %q, want %q", state.metadata.Iceberg.TableUUID, prior.metadata.Iceberg.TableUUID)
	}
}

// TestCleanupSafeMetadataKeepsFailedRollback proves a flip failure at index 2
// where table 0's rollback also fails leaves table 0's new metadata in place
// (its pointer still references it), while the rolled-back table 1 and the
// never-flipped tables 2 and 3 are safe to delete.
func TestCleanupSafeMetadataKeepsFailedRollback(t *testing.T) {
	prepared := make([]preparedTableCommit, 4)
	rolledBack := []bool{false, true} // tables 0 and 1 were flipped; 0's rollback failed

	safe := cleanupSafeMetadata(prepared, 2, rolledBack)

	want := []bool{false, true, true, true}
	for i := range want {
		if safe[i] != want[i] {
			t.Fatalf("safe[%d] = %v, want %v (full %v)", i, safe[i], want[i], safe)
		}
	}
}

// TestCleanupSafeMetadataFirstFlipFails covers a flip failure at the first table:
// nothing was flipped, so every written file is safe to delete.
func TestCleanupSafeMetadataFirstFlipFails(t *testing.T) {
	prepared := make([]preparedTableCommit, 3)
	safe := cleanupSafeMetadata(prepared, 0, nil)
	for i, ok := range safe {
		if !ok {
			t.Fatalf("safe[%d] = false, want true (full %v)", i, safe)
		}
	}
}

// TestCleanupSafeMetadataAllRolledBack covers the common case where every prior
// flip rolls back cleanly, so all metadata files are safe to delete.
func TestCleanupSafeMetadataAllRolledBack(t *testing.T) {
	prepared := make([]preparedTableCommit, 3)
	safe := cleanupSafeMetadata(prepared, 2, []bool{true, true})
	for i, ok := range safe {
		if !ok {
			t.Fatalf("safe[%d] = false, want true (full %v)", i, safe)
		}
	}
}

// tableEntry models the persisted table fields and the partial-update rules
// handleUpdateTable applies, so rollback can be exercised without a live filer.
type tableEntry struct {
	metadataLocation string
	metadataVersion  int
	metadata         *s3tables.TableMetadata
}

func (e tableEntry) clone() *tableEntry {
	return &tableEntry{
		metadataLocation: e.metadataLocation,
		metadataVersion:  e.metadataVersion,
		metadata:         cloneTableMetadata(e.metadata),
	}
}

func (e *tableEntry) applyUpdate(req *s3tables.UpdateTableRequest) {
	if req.Metadata != nil {
		if e.metadata == nil {
			e.metadata = &s3tables.TableMetadata{}
		}
		if req.Metadata.Iceberg != nil {
			if e.metadata.Iceberg == nil {
				e.metadata.Iceberg = &s3tables.IcebergMetadata{}
			}
			if req.Metadata.Iceberg.TableUUID != "" {
				e.metadata.Iceberg.TableUUID = req.Metadata.Iceberg.TableUUID
			}
		}
		if len(req.Metadata.FullMetadata) > 0 {
			e.metadata.FullMetadata = req.Metadata.FullMetadata
		}
	}
	if req.MetadataLocation != "" {
		e.metadataLocation = req.MetadataLocation
	}
	if req.MetadataVersion > 0 {
		e.metadataVersion = req.MetadataVersion
	}
}

// A transaction against a registration that kept only its metadata location
// reads that file: the UUID assertion the client took from its load passes,
// and the commit lands on the registered snapshots instead of a placeholder.
func TestCommitTransactionReadsMetadataFileOfOlderRegistration(t *testing.T) {
	fc := newMemFiler()
	seedNamespace(fc, registerTestBucket, "finance", "alice")
	s := NewServer(fc, nil)
	location, registered := seedRegistrableMetadata(t, fc, "events")

	err := fc.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		return s.tablesManager.Execute(context.Background(), s3tables.NewManagerClient(client), "RegisterTable", &s3tables.RegisterTableRequest{
			TableBucketARN:   buildTableBucketARN(registerTestBucket),
			Namespace:        []string{"finance"},
			Name:             "events",
			MetadataLocation: location,
		}, nil, "alice")
	})
	if err != nil {
		t.Fatalf("seed older registration: %v", err)
	}

	commitTransaction := func(requirements, updates string) *httptest.ResponseRecorder {
		body := `{"table-changes":[{"identifier":{"namespace":["finance"],"name":"events"},` +
			`"requirements":[` + requirements + `],"updates":[` + updates + `]}]}`
		r := httptest.NewRequest(http.MethodPost, "/v1/"+registerTestBucket+"/transactions/commit", strings.NewReader(body))
		r = mux.SetURLVars(r, map[string]string{"prefix": registerTestBucket})
		r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
		w := httptest.NewRecorder()
		s.handleCommitTransaction(w, r)
		return w
	}

	w := commitTransaction(
		`{"type":"assert-table-uuid","uuid":"`+registered.TableUUID().String()+`"}`,
		`{"action":"set-properties","updates":{"committed":"true"}}`)
	if w.Code != http.StatusNoContent {
		t.Fatalf("uuid-asserting transaction: status = %d, want %d (body: %s)", w.Code, http.StatusNoContent, w.Body.String())
	}

	w = commitTransaction("", `{"action":"set-properties","updates":{"again":"true"}}`)
	if w.Code != http.StatusNoContent {
		t.Fatalf("requirement-free transaction: status = %d, want %d (body: %s)", w.Code, http.StatusNoContent, w.Body.String())
	}

	w = httptest.NewRecorder()
	s.handleLoadTable(w, registeredTableRequest(http.MethodGet, "events", ""))
	loaded := servedMetadata(t, "load after transactions", w)
	checkServesRegistered(t, "load after transactions", loaded, registered)
	if got := loaded.Properties()["committed"]; got != "true" {
		t.Errorf("property committed = %q, want true", got)
	}
	if got := loaded.Properties()["again"]; got != "true" {
		t.Errorf("property again = %q, want true", got)
	}
}

// A transaction re-stamps the default name mapping the way a single-table
// commit does, so readers of field-id-less files keep resolving columns even
// when the table's record started without one.
func TestCommitTransactionRefreshesNameMapping(t *testing.T) {
	fc := newMemFiler()
	seedNamespace(fc, registerTestBucket, "finance", "alice")
	s := NewServer(fc, nil)

	schema := iceberg.NewSchema(0,
		iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
		iceberg.NestedField{ID: 2, Name: "label", Type: iceberg.PrimitiveTypes.String},
	)
	metadata, err := newTableMetadata(uuid.New(), "s3://"+registerTestBucket+"/finance/events", schema, nil, nil, nil)
	if err != nil {
		t.Fatalf("newTableMetadata: %v", err)
	}
	raw, err := json.Marshal(metadata)
	if err != nil {
		t.Fatalf("marshal metadata: %v", err)
	}
	var doc map[string]any
	if err := json.Unmarshal(raw, &doc); err != nil {
		t.Fatalf("decode metadata: %v", err)
	}
	if props, ok := doc["properties"].(map[string]any); ok {
		delete(props, table.DefaultNameMappingKey)
	}
	if raw, err = json.Marshal(doc); err != nil {
		t.Fatalf("re-marshal metadata: %v", err)
	}
	location := seedMetadataFile(fc, "events", "00001-7c1e2f4a.metadata.json", raw)

	err = fc.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		return s.tablesManager.Execute(context.Background(), s3tables.NewManagerClient(client), "RegisterTable", &s3tables.RegisterTableRequest{
			TableBucketARN:   buildTableBucketARN(registerTestBucket),
			Namespace:        []string{"finance"},
			Name:             "events",
			MetadataLocation: location,
		}, nil, "alice")
	})
	if err != nil {
		t.Fatalf("seed older registration: %v", err)
	}

	body := `{"table-changes":[{"identifier":{"namespace":["finance"],"name":"events"},` +
		`"updates":[{"action":"set-properties","updates":{"committed":"true"}}]}]}`
	r := httptest.NewRequest(http.MethodPost, "/v1/"+registerTestBucket+"/transactions/commit", strings.NewReader(body))
	r = mux.SetURLVars(r, map[string]string{"prefix": registerTestBucket})
	r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
	w := httptest.NewRecorder()
	s.handleCommitTransaction(w, r)
	if w.Code != http.StatusNoContent {
		t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusNoContent, w.Body.String())
	}

	w = httptest.NewRecorder()
	s.handleLoadTable(w, registeredTableRequest(http.MethodGet, "events", ""))
	loaded := servedMetadata(t, "load after transaction", w)
	mappingJSON, ok := loaded.Properties()[table.DefaultNameMappingKey]
	if !ok || mappingJSON == "" {
		t.Fatalf("committed metadata lost %s: %v", table.DefaultNameMappingKey, loaded.Properties())
	}
}

// failUpdateFiler fails entry updates on one path, so a transaction's flip of
// that table fails after the tables before it in the request have flipped.
type failUpdateFiler struct {
	*memFiler
	failPath string
}

func (f *failUpdateFiler) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
	return fn(f)
}

func (f *failUpdateFiler) UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
	if path.Join(in.Directory, in.Entry.Name) == f.failPath {
		return nil, errors.New("injected update failure")
	}
	return f.memFiler.UpdateEntry(ctx, in, opts...)
}

// A transaction that fails after flipping a table rolls it back. UpdateTable
// cannot clear the FullMetadata a flip stored, so for a registration that kept
// only its metadata location the restore has to write back the file's
// metadata: otherwise loads go on serving the change the client was told
// failed, and a client retrying the transaction applies it twice.
func TestCommitTransactionRollbackRestoresOlderRegistration(t *testing.T) {
	fc := &failUpdateFiler{memFiler: newMemFiler()}
	seedNamespace(fc.memFiler, registerTestBucket, "finance", "alice")
	s := NewServer(fc, nil)
	location, registered := seedRegistrableMetadata(t, fc.memFiler, "events")

	err := fc.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		return s.tablesManager.Execute(context.Background(), s3tables.NewManagerClient(client), "RegisterTable", &s3tables.RegisterTableRequest{
			TableBucketARN:   buildTableBucketARN(registerTestBucket),
			Namespace:        []string{"finance"},
			Name:             "events",
			MetadataLocation: location,
		}, nil, "alice")
	})
	if err != nil {
		t.Fatalf("seed older registration: %v", err)
	}
	otherLocation, _ := seedRegistrableMetadata(t, fc.memFiler, "other")
	w := httptest.NewRecorder()
	s.handleRegisterTable(w, registerTableRequest("other", otherLocation))
	if w.Code != http.StatusOK {
		t.Fatalf("register other: status = %d (body: %s)", w.Code, w.Body.String())
	}

	// events flips first; the flip of other then fails and rolls events back.
	fc.failPath = s3tables.GetTablePath(registerTestBucket, "finance", "other")
	change := func(name string) string {
		return `{"identifier":{"namespace":["finance"],"name":"` + name + `"},` +
			`"updates":[{"action":"set-properties","updates":{"committed":"true"}}]}`
	}
	body := `{"table-changes":[` + change("events") + `,` + change("other") + `]}`
	r := httptest.NewRequest(http.MethodPost, "/v1/"+registerTestBucket+"/transactions/commit", strings.NewReader(body))
	r = mux.SetURLVars(r, map[string]string{"prefix": registerTestBucket})
	r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
	w = httptest.NewRecorder()
	s.handleCommitTransaction(w, r)
	if w.Code == http.StatusNoContent {
		t.Fatal("transaction succeeded although the flip of other failed")
	}

	w = httptest.NewRecorder()
	s.handleLoadTable(w, registeredTableRequest(http.MethodGet, "events", ""))
	loaded := servedMetadata(t, "load after rolled-back transaction", w)
	checkServesRegistered(t, "load after rolled-back transaction", loaded, registered)
	if got, ok := loaded.Properties()["committed"]; ok {
		t.Errorf("load after rolled-back transaction serves its property committed = %q", got)
	}
}

// A transaction on a registration kept by location alone — one whose metadata
// is too large to ship — flips the pointer by location too, and the record
// holds no stored copy that would shadow the staged file.
func TestCommitTransactionOnOversizedRegistration(t *testing.T) {
	fc := newMemFiler()
	seedNamespace(fc, registerTestBucket, "finance", "alice")
	s := NewServer(fc, nil)
	location, registered := seedOversizedMetadata(t, fc, "events")

	err := fc.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		return s.tablesManager.Execute(context.Background(), s3tables.NewManagerClient(client), "RegisterTable", &s3tables.RegisterTableRequest{
			TableBucketARN:   buildTableBucketARN(registerTestBucket),
			Namespace:        []string{"finance"},
			Name:             "events",
			MetadataLocation: location,
		}, nil, "alice")
	})
	if err != nil {
		t.Fatalf("seed location-only registration: %v", err)
	}

	body := `{"table-changes":[{"identifier":{"namespace":["finance"],"name":"events"},` +
		`"requirements":[{"type":"assert-table-uuid","uuid":"` + registered.TableUUID().String() + `"}],` +
		`"updates":[{"action":"set-properties","updates":{"committed":"true"}}]}]}`
	r := httptest.NewRequest(http.MethodPost, "/v1/"+registerTestBucket+"/transactions/commit", strings.NewReader(body))
	r = mux.SetURLVars(r, map[string]string{"prefix": registerTestBucket})
	r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice"))
	w := httptest.NewRecorder()
	s.handleCommitTransaction(w, r)
	if w.Code != http.StatusNoContent {
		t.Fatalf("oversized transaction: status = %d, want %d (body: %s)", w.Code, http.StatusNoContent, w.Body.String())
	}

	if stored := storedMetadata(t, s, fc, "events"); stored != nil && len(stored.FullMetadata) != 0 {
		t.Errorf("catalog record holds %d bytes of metadata, over the manager's request limit", len(stored.FullMetadata))
	}

	w = httptest.NewRecorder()
	s.handleLoadTable(w, registeredTableRequest(http.MethodGet, "events", ""))
	loaded := servedMetadata(t, "load after transaction", w)
	checkServesRegistered(t, "load after transaction", loaded, registered)
	if got := loaded.Properties()["committed"]; got != "true" {
		t.Errorf("load after transaction: property committed = %q, want true", got)
	}
}