Files
seaweedfs/weed/s3api/iceberg/handlers_namespace.go
T
Chris Lu 761ec7da00 fix(iceberg): use dot separator for namespace paths instead of unit separator (#8960)
* fix(iceberg): use dot separator for namespace paths instead of unit separator

The Iceberg REST Catalog handler was using \x1F (unit separator) to join
multi-level namespaces when constructing S3 location and filer paths. The
S3 Tables storage layer uses "." (dot) as the namespace separator, causing
tables created via the Iceberg REST API to point to different paths than
where S3 Tables actually stores them.

Fixes #8959

* fix(iceberg): use dot separator in log messages for readable namespace output

* fix(iceberg): use path.Join for S3 location path segments

Use path.Join to construct the namespace/table path segments in fallback
S3 locations for robustness and consistency with handleCreateTable.

* test(iceberg): add multi-level namespace integration tests for Spark and Trino

Add regression tests for #8959 that create a two-level namespace (e.g.
"analytics.daily"), create a table under it, insert data, and query it
back. This exercises the dot-separated namespace path construction and
verifies that Spark/Trino can actually read the data at the S3 location
returned by the Iceberg REST API.

* fix(test): enable nested namespace in Trino Iceberg catalog config

Trino requires `iceberg.rest-catalog.nested-namespace-enabled=true` to
support multi-level namespaces. Without this, CREATE SCHEMA with a
dotted name fails with "Nested namespace is not enabled for this catalog".

* fix(test): parse Trino COUNT(*) output as integer instead of substring match

Avoids false matches from strings.Contains(output, "3") by parsing the
actual numeric result with strconv.Atoi and asserting equality.

* fix(test): use separate Trino config for nested namespace test

The nested-namespace-enabled=true setting in Trino changes how SHOW
SCHEMAS works, causing "Internal error" for all tests sharing that
catalog config. Move the flag to a dedicated config used only by
TestTrinoMultiLevelNamespace.

* fix(iceberg): support parent query parameter in ListNamespaces for nested namespaces

Add handling for the Iceberg REST spec's `parent` query parameter in
handleListNamespaces. When Trino has nested-namespace-enabled=true, it
sends `GET /v1/namespaces?parent=<ns>` to list child namespaces. The
parent value is decoded from the Iceberg unit separator format and
converted to a dot-separated prefix for the S3 Tables layer.

Also simplify TestTrinoMultiLevelNamespace to focus on namespace
operations (create, list, show tables) rather than data operations,
since Trino's REST catalog has a non-empty location check that conflicts
with server-side metadata creation.

* fix(test): expand Trino multi-level namespace test and merge config helpers

- Expand TestTrinoMultiLevelNamespace to create a table with explicit
  location, insert rows, query them back, and verify the S3 file path
  contains the dot-separated namespace (not \x1F). This ensures the
  original #8959 bug would be caught by the Trino integration test.
- Merge writeTrinoConfig and writeTrinoNestedNamespaceConfig into a
  single parameterized function using functional options.
2026-04-07 12:21:22 -07:00

259 lines
8.2 KiB
Go

package iceberg
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// handleConfig returns catalog configuration.
func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
config := CatalogConfig{
Defaults: map[string]string{},
Overrides: map[string]string{},
}
if err := json.NewEncoder(w).Encode(config); err != nil {
glog.Warningf("handleConfig: Failed to encode config: %v", err)
}
}
// handleListNamespaces lists namespaces in a catalog.
func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
pageToken, pageSize, err := parsePagination(r)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
return
}
// The Iceberg REST spec allows a "parent" query parameter for hierarchical
// namespace listing. Convert it to the dot-separated prefix used by S3 Tables.
var prefix string
if parent := r.URL.Query().Get("parent"); parent != "" {
parentParts := parseNamespace(parent)
if len(parentParts) > 0 {
prefix = flattenNamespacePath(parentParts) + "."
}
}
// Use S3 Tables manager to list namespaces
var resp s3tables.ListNamespacesResponse
req := &s3tables.ListNamespacesRequest{
TableBucketARN: bucketARN,
Prefix: prefix,
ContinuationToken: pageToken,
MaxNamespaces: pageSize,
}
err = s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "ListNamespaces", req, &resp, identityName)
})
if err != nil {
glog.Infof("Iceberg: ListNamespaces error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
// Convert to Iceberg format
namespaces := make([]Namespace, 0, len(resp.Namespaces))
for _, ns := range resp.Namespaces {
namespaces = append(namespaces, Namespace(ns.Namespace))
}
result := ListNamespacesResponse{
NextPageToken: resp.ContinuationToken,
Namespaces: namespaces,
}
writeJSON(w, http.StatusOK, result)
}
// handleCreateNamespace creates a new namespace.
func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
var req CreateNamespaceRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body")
return
}
if len(req.Namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
// Use S3 Tables manager to create namespace
createReq := &s3tables.CreateNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: req.Namespace,
Properties: normalizeNamespaceProperties(req.Properties),
}
var createResp s3tables.CreateNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
glog.V(2).Infof("Iceberg: handleCreateNamespace calling Execute with identityName=%s", identityName)
return s.tablesManager.Execute(r.Context(), mgrClient, "CreateNamespace", createReq, &createResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "already exists") {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
glog.Errorf("Iceberg: CreateNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
result := CreateNamespaceResponse{
Namespace: Namespace(createResp.Namespace),
Properties: normalizeNamespaceProperties(createResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleGetNamespace gets namespace metadata.
func (s *Server) handleGetNamespace(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
// Use S3 Tables manager to get namespace
getReq := &s3tables.GetNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
var getResp s3tables.GetNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
return
}
glog.V(1).Infof("Iceberg: GetNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
result := GetNamespaceResponse{
Namespace: Namespace(getResp.Namespace),
Properties: normalizeNamespaceProperties(getResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleNamespaceExists checks if a namespace exists.
func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
getReq := &s3tables.GetNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
var getResp s3tables.GetNamespaceResponse
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetNamespace", getReq, &getResp, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
w.WriteHeader(http.StatusNotFound)
return
}
glog.V(1).Infof("Iceberg: NamespaceExists error: %v", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
// handleDropNamespace deletes a namespace.
func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
namespace := parseNamespace(vars["namespace"])
if len(namespace) == 0 {
writeError(w, http.StatusBadRequest, "BadRequestException", "Namespace is required")
return
}
bucketName := getBucketFromPrefix(r)
bucketARN := buildTableBucketARN(bucketName)
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
deleteReq := &s3tables.DeleteNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
}
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "DeleteNamespace", deleteReq, nil, identityName)
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", fmt.Sprintf("Namespace does not exist: %v", namespace))
return
}
if strings.Contains(err.Error(), "not empty") {
writeError(w, http.StatusConflict, "NamespaceNotEmptyException", "Namespace is not empty")
return
}
glog.V(1).Infof("Iceberg: DropNamespace error: %v", err)
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}