seaweedfs/weed/s3api/iceberg/iceberg_create_view_test.go

package iceberg

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"path"
	"sort"
	"strings"
	"testing"

	"github.com/apache/iceberg-go"
	"github.com/apache/iceberg-go/table"
	"github.com/apache/iceberg-go/view"
	"github.com/gorilla/mux"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// memFiler is a minimal in-memory SeaweedFilerClient backing the view handlers
// end-to-end through the s3tables manager. Only the entry operations the create
// path touches are implemented; anything else panics so a missing dependency is
// loud rather than silently passing.
type memFiler struct {
	filer_pb.SeaweedFilerClient
	entries map[string]*filer_pb.Entry
	// failFileCreate, when set, makes CreateEntry fail for non-directory entries,
	// simulating a metadata-file write error.
	failFileCreate error
	// onLookup, when set, runs at the start of every LookupDirectoryEntry.
	onLookup func(p string)
}

func newMemFiler() *memFiler {
	return &memFiler{entries: map[string]*filer_pb.Entry{}}
}

func (m *memFiler) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
	return fn(m)
}

func (m *memFiler) seed(p string, entry *filer_pb.Entry) {
	m.entries[p] = entry
}

func (m *memFiler) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
	if hook := m.onLookup; hook != nil {
		hook(path.Join(in.Directory, in.Name))
	}
	entry, ok := m.entries[path.Join(in.Directory, in.Name)]
	if !ok {
		return nil, filer_pb.ErrNotFound
	}
	return &filer_pb.LookupDirectoryEntryResponse{Entry: entry}, nil
}

func (m *memFiler) CreateEntry(_ context.Context, in *filer_pb.CreateEntryRequest, _ ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
	if m.failFileCreate != nil && !in.Entry.IsDirectory {
		return nil, m.failFileCreate
	}
	if _, taken := m.entries[path.Join(in.Directory, in.Entry.Name)]; taken && in.OExcl {
		return &filer_pb.CreateEntryResponse{ErrorCode: filer_pb.FilerError_ENTRY_ALREADY_EXISTS}, nil
	}
	m.entries[path.Join(in.Directory, in.Entry.Name)] = in.Entry
	return &filer_pb.CreateEntryResponse{}, nil
}

func (m *memFiler) DeleteEntry(_ context.Context, in *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
	prefix := path.Join(in.Directory, in.Name)
	if entry, ok := m.entries[prefix]; ok {
		for key, want := range in.ExpectedExtended {
			if string(entry.Extended[key]) != string(want) {
				return nil, status.Errorf(codes.FailedPrecondition, "%s: %s changed", prefix, key)
			}
		}
	}
	for p := range m.entries {
		if p == prefix || strings.HasPrefix(p, prefix+"/") {
			delete(m.entries, p)
		}
	}
	return &filer_pb.DeleteEntryResponse{}, nil
}

func (m *memFiler) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
	m.entries[path.Join(in.Directory, in.Entry.Name)] = in.Entry
	return &filer_pb.UpdateEntryResponse{}, nil
}

func (m *memFiler) ListEntries(_ context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_ListEntriesClient, error) {
	var names []string
	for p := range m.entries {
		if path.Dir(p) != in.Directory {
			continue
		}
		if name := path.Base(p); in.StartFromFileName != "" && (name < in.StartFromFileName || name == in.StartFromFileName && !in.InclusiveStartFrom) {
			continue
		}
		names = append(names, p)
	}
	sort.Strings(names)
	stream := &memListEntries{}
	for _, name := range names {
		if in.Limit > 0 && uint32(len(stream.entries)) >= in.Limit {
			break
		}
		stream.entries = append(stream.entries, m.entries[name])
	}
	return stream, nil
}

type memListEntries struct {
	grpc.ClientStream
	entries []*filer_pb.Entry
}

func (m *memListEntries) Recv() (*filer_pb.ListEntriesResponse, error) {
	if len(m.entries) == 0 {
		return nil, io.EOF
	}
	entry := m.entries[0]
	m.entries = m.entries[1:]
	return &filer_pb.ListEntriesResponse{Entry: entry}, nil
}

func newCreateViewRequest(t *testing.T, namespace, name, sql string) *http.Request {
	t.Helper()
	schema := iceberg.NewSchemaWithIdentifiers(0, nil,
		iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true},
	)
	ver, err := view.NewVersionFromSQL(1, schema.ID, sql, table.Identifier{namespace})
	if err != nil {
		t.Fatal(err)
	}
	body, err := json.Marshal(CreateViewRequest{Name: name, Schema: schema, ViewVersion: ver})
	if err != nil {
		t.Fatal(err)
	}
	r := httptest.NewRequest(http.MethodPost, "/v1/namespaces/"+namespace+"/views", bytes.NewReader(body))
	r = mux.SetURLVars(r, map[string]string{"namespace": namespace})
	r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), s3_constants.AccountAdminId))
	return r
}

// seedNamespace registers a bucket and namespace owned by owner so the s3tables
// existence and auth-context lookups pass.
func seedNamespace(fc *memFiler, bucket, namespace, owner string) {
	fc.seed(s3tables.GetTableBucketPath(bucket), &filer_pb.Entry{Name: bucket, IsDirectory: true})
	meta, _ := json.Marshal(map[string]any{"namespace": []string{namespace}, "ownerAccountId": owner})
	fc.seed(s3tables.GetNamespacePath(bucket, namespace), &filer_pb.Entry{
		Name:        namespace,
		IsDirectory: true,
		Extended:    map[string][]byte{s3tables.ExtendedKeyMetadata: meta},
	})
}

func TestCreateViewMissingNamespaceReturns404(t *testing.T) {
	fc := newMemFiler()
	s := NewServer(fc, nil)

	w := httptest.NewRecorder()
	s.handleCreateView(w, newCreateViewRequest(t, "ns", "v", "SELECT 1"))

	if w.Code != http.StatusNotFound {
		t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusNotFound, w.Body.String())
	}
	var errResp ErrorResponse
	if err := json.Unmarshal(w.Body.Bytes(), &errResp); err != nil {
		t.Fatalf("unmarshal error body: %v", err)
	}
	if errResp.Error.Type != "NoSuchNamespaceException" {
		t.Fatalf("error type = %q, want NoSuchNamespaceException", errResp.Error.Type)
	}
}

func TestCreateViewTagsEntryAsView(t *testing.T) {
	const bucket = "warehouse"
	fc := newMemFiler()
	seedNamespace(fc, bucket, "ns", s3_constants.AccountAdminId)
	s := NewServer(fc, nil)

	w := httptest.NewRecorder()
	s.handleCreateView(w, newCreateViewRequest(t, "ns", "v", "SELECT 1"))
	if w.Code != http.StatusOK {
		t.Fatalf("create status = %d, want 200 (body: %s)", w.Code, w.Body.String())
	}

	entry, ok := fc.entries[s3tables.GetTablePath(bucket, "ns", "v")]
	if !ok {
		t.Fatalf("view entry not created")
	}
	if got := string(entry.Extended[s3tables.ExtendedKeyEntryType]); got != s3tables.EntryTypeView {
		t.Fatalf("entryType = %q, want %q", got, s3tables.EntryTypeView)
	}
	if _, ok := entry.Extended[s3tables.ExtendedKeyMetadata]; !ok {
		t.Fatalf("view entry missing metadata attribute")
	}
}

func TestCreateViewDuplicateDoesNotClobberMetadata(t *testing.T) {
	const bucket = "warehouse"
	fc := newMemFiler()
	seedNamespace(fc, bucket, "ns", s3_constants.AccountAdminId)
	s := NewServer(fc, nil)

	w := httptest.NewRecorder()
	s.handleCreateView(w, newCreateViewRequest(t, "ns", "v", "SELECT 1"))
	if w.Code != http.StatusOK {
		t.Fatalf("first create status = %d, want 200 (body: %s)", w.Code, w.Body.String())
	}

	var created struct {
		MetadataLocation string `json:"metadata-location"`
	}
	if err := json.Unmarshal(w.Body.Bytes(), &created); err != nil {
		t.Fatalf("decode create response: %v", err)
	}
	if !strings.HasPrefix(created.MetadataLocation, "s3://"+bucket+"/ns/v-") {
		t.Fatalf("metadata-location = %q, want one under a directory of the view's own", created.MetadataLocation)
	}
	metadataKey := path.Join(s3tables.TablesPath, strings.TrimPrefix(created.MetadataLocation, "s3://"))
	first, ok := fc.entries[metadataKey]
	if !ok {
		t.Fatalf("metadata file not written at %s", metadataKey)
	}
	original := append([]byte(nil), first.Content...)

	// Re-create the same view with different SQL: the existence pre-check must
	// short-circuit so the persisted v1.metadata.json stays byte-for-byte intact.
	w = httptest.NewRecorder()
	s.handleCreateView(w, newCreateViewRequest(t, "ns", "v", "SELECT 2"))
	if w.Code != http.StatusOK {
		t.Fatalf("duplicate create status = %d, want 200 (body: %s)", w.Code, w.Body.String())
	}
	if got := fc.entries[metadataKey].Content; !bytes.Equal(got, original) {
		t.Fatalf("duplicate create clobbered stored metadata:\n got %s\nwant %s", got, original)
	}
}

// A view create that loses the race for its name answers with the view the
// catalog registered and writes no metadata of its own.
func TestCreateViewThatLosesTheRaceReturnsTheRegisteredView(t *testing.T) {
	const bucket = "warehouse"
	fc := newMemFiler()
	seedNamespace(fc, bucket, "ns", s3_constants.AccountAdminId)
	s := NewServer(fc, nil)
	first := httptest.NewRecorder()
	raceAtSecondLookup(fc, s3tables.GetTablePath(bucket, "ns", "v"), func() {
		s.handleCreateView(first, newCreateViewRequest(t, "ns", "v", "SELECT 1"))
	})
	second := httptest.NewRecorder()
	s.handleCreateView(second, newCreateViewRequest(t, "ns", "v", "SELECT 2"))

	won, lost := decodeCreated(t, first), decodeCreated(t, second)
	if lost != won {
		t.Errorf("losing create answered %+v, want the registered view %+v", lost, won)
	}
	for p, entry := range fc.entries {
		if !entry.IsDirectory && strings.HasSuffix(p, ".metadata.json") && p != path.Join(s3tables.TablesPath, strings.TrimPrefix(won.MetadataLocation, "s3://")) {
			t.Errorf("losing create wrote %s", p)
		}
	}
}

func TestCreateViewRollsBackEntryWhenMetadataWriteFails(t *testing.T) {
	const bucket = "warehouse"
	fc := newMemFiler()
	seedNamespace(fc, bucket, "ns", s3_constants.AccountAdminId)
	fc.failFileCreate = errors.New("disk full")
	s := NewServer(fc, nil)

	w := httptest.NewRecorder()
	s.handleCreateView(w, newCreateViewRequest(t, "ns", "v", "SELECT 1"))
	if w.Code != http.StatusInternalServerError {
		t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusInternalServerError, w.Body.String())
	}

	// The registered view must be rolled back so it doesn't linger pointing at
	// metadata that was never written.
	if _, ok := fc.entries[s3tables.GetTablePath(bucket, "ns", "v")]; ok {
		t.Fatalf("view entry left behind after metadata write failure")
	}
}