seaweedfs/weed/s3api/s3tables/handler_warehouse_location_test.go

package s3tables

import (
	"context"
	"errors"
	"net/http"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
)

// A table created without a location keeps its files in a directory of its
// own, so a rename leaves nothing under the old name and the name can be used
// again without either table reaching the other's files.
func TestCreateTableWithoutLocationGetsItsOwnDirectory(t *testing.T) {
	fs, m := startRenameManager(t)
	bucketARN := mustBucketARN(t)
	nsPath := GetNamespacePath(renameTestBucket, "ns")

	requireOK(t, m, fs, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "w", Format: FormatIceberg})
	var table GetTableResponse
	getTable(t, m, fs, bucketARN, "w", &table)
	assert.Empty(t, table.MetadataLocation)
	assert.True(t, strings.HasPrefix(table.WarehouseLocation, renameLocation("ns/w-")), table.WarehouseLocation)

	firstRoot := recordRoot(t, fs, nsPath, "w")
	assert.Equal(t, path.Join(TablesPath, strings.TrimPrefix(table.WarehouseLocation, "s3://")), firstRoot)
	require.NotNil(t, fs.Get(firstRoot, "data"))
	fs.PutFile(path.Join(firstRoot, "data"), "part-0.parquet", time.Now())
	assert.Nil(t, fs.Get(path.Join(nsPath, "w"), "data"), "the catalog entry holds no files")

	requireOK(t, m, fs, "RenameTable", &RenameTableRequest{TableBucketARN: bucketARN, SourceNamespace: []string{"ns"}, SourceName: "w", DestNamespace: []string{"ns"}, DestName: "moved"})
	assert.Equal(t, firstRoot, recordRoot(t, fs, nsPath, "moved"), "a rename keeps the directory")

	requireOK(t, m, fs, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "w", Format: FormatIceberg})
	secondRoot := recordRoot(t, fs, nsPath, "w")
	assert.NotEqual(t, firstRoot, secondRoot)
	assert.False(t, PathsOverlap(firstRoot, secondRoot))

	requireOK(t, m, fs, "DeleteTable", &DeleteTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "w"})
	assert.NotNil(t, fs.Get(path.Join(firstRoot, "data"), "part-0.parquet"), "deleting the new table leaves the renamed one's files")
	requireOK(t, m, fs, "DeleteTable", &DeleteTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "moved"})
	assert.Nil(t, fs.Get(path.Dir(firstRoot), path.Base(firstRoot)), "deleting the renamed table purges its directory")
}

// A Lance dataset stays at its name path, where Lance clients and the vector
// engine look for it.
func TestCreateLanceTableWithoutLocationKeepsItsPath(t *testing.T) {
	fs, m := startRenameManager(t)
	bucketARN := mustBucketARN(t)
	requireOK(t, m, fs, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "l", Format: FormatLance})
	var table GetTableResponse
	getTable(t, m, fs, bucketARN, "l", &table)
	assert.Empty(t, table.MetadataLocation)
	assert.Empty(t, table.WarehouseLocation)
	assert.Equal(t, GetTablePath(renameTestBucket, "ns", "l"), recordRoot(t, fs, GetNamespacePath(renameTestBucket, "ns"), "l"))
}

// Records written before tables got their own directory carry no warehouse
// location and keep resolving to their name path, so nothing has to be
// rewritten when a gateway is upgraded.
func TestRecordWithoutWarehouseLocationKeepsItsPath(t *testing.T) {
	fs, m := startRenameManager(t)
	bucketARN := mustBucketARN(t)
	nsPath := GetNamespacePath(renameTestBucket, "ns")
	requireOK(t, m, fs, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "old", Format: FormatIceberg})
	dropWarehouseLocation(t, fs, nsPath, "old")
	assert.Equal(t, path.Join(nsPath, "old"), recordRoot(t, fs, nsPath, "old"))
	assert.Equal(t, path.Join(nsPath, "t"), recordRoot(t, fs, nsPath, "t"), "a metadata location still decides the root")

	status, resp := runHandler(t, m, fs.Client, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "alias", Format: FormatIceberg, MetadataLocation: renameLocation("ns/old/metadata/v1.metadata.json")})
	assert.Equal(t, http.StatusConflict, status, "%+v", resp)
}

// The first commit of a table writes its metadata into the assigned directory,
// which is the root the record already holds, so it scans no other record.
func TestFirstCommitIntoWarehouseLocationKeepsTheRoot(t *testing.T) {
	fs, m := startRenameManager(t)
	bucketARN := mustBucketARN(t)
	nsPath := GetNamespacePath(renameTestBucket, "ns")
	requireOK(t, m, fs, "CreateTable", &CreateTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "w", Format: FormatIceberg})
	var table GetTableResponse
	getTable(t, m, fs, bucketARN, "w", &table)
	root := recordRoot(t, fs, nsPath, "w")

	status, resp := runHandler(t, m, &noListClient{SeaweedFilerClient: fs.Client}, "UpdateTable", &UpdateTableRequest{
		TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: "w", MetadataVersion: 1,
		MetadataLocation: table.WarehouseLocation + "/metadata/v1.metadata.json",
	})
	require.Equal(t, http.StatusOK, status, "%+v", resp)
	assert.Equal(t, root, recordRoot(t, fs, nsPath, "w"))
	assert.NotContains(t, fs.Get(nsPath, "w").Extended, ExtendedKeyMoving)
}

// A create that backs out after making the table's directory removes it, so
// failed creates leave nothing behind under the namespace.
func TestBackedOutCreateRemovesItsDirectory(t *testing.T) {
	fs, m := startEmptyBucket(t)
	requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t))
	bucketPath := GetTableBucketPath(renameTestBucket)
	nsPath := GetNamespacePath(renameTestBucket, "ns")
	recreated := recreatedNamespace(t, "ns")
	swapped := false
	fs.OnLookup = func(dir, name string) {
		switch {
		case dir == nsPath && name == "t" && !swapped:
			swapped = true
			fs.Put(bucketPath, "ns", recreated)
		case path.Dir(dir) == nsPath && name == "data":
			// The filer makes the parents of an entry it creates.
			fs.Put(nsPath, path.Base(dir), nil)
		}
	}

	code, resp := runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t"))
	require.Equal(t, http.StatusNotFound, code, "%+v", resp)
	fs.OnLookup = nil

	var left []string
	require.NoError(t, eachDirEntry(context.Background(), fs.Client, nsPath, func(entry *filer_pb.Entry) bool {
		left = append(left, entry.Name)
		return true
	}))
	assert.Empty(t, left)
}

func getTable(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, bucketARN, name string, out *GetTableResponse) {
	t.Helper()
	require.NoError(t, m.Execute(context.Background(), NewManagerClient(fs.Client), "GetTable", &GetTableRequest{TableBucketARN: bucketARN, Namespace: []string{"ns"}, Name: name}, out, ""))
}

type noListClient struct {
	filer_pb.SeaweedFilerClient
}

func (c *noListClient) ListEntries(context.Context, *filer_pb.ListEntriesRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
	return nil, errors.New("listing unavailable")
}