mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-19 04:50:54 +02:00
* iceberg: withhold the S3 endpoint from credential-vending clients A client that sends X-Iceberg-Access-Delegation: vended-credentials builds its storage credential out of the LoadTable config and drops the one it was configured with. We vend no credentials, so the endpoint we advertised left DuckDB signing nothing: every metadata and data file came back 403, and its attempt to refresh the empty credential 404ed on stage-created tables. Answer those clients with no config at all so they keep their own credentials. Clients that do not ask for delegation still get the endpoint. * iceberg: mark load responses as varying on the delegation header The FileIO config in a table or view load response now depends on whether the client asked for vended credentials, so a cache between us and the client must key on that header rather than on the URL alone. * test: cover the DuckDB vended-credentials access pattern Runs weed mini with -s3.externalUrl, which is what makes the catalog advertise an endpoint at all, and checks both halves: a plain LoadTable still gets the endpoint, while one asking for vended credentials never gets an endpoint without the credentials to sign with. The DuckDB round trip creates a table from a query and reads it back, which is the flow that failed with 403 on every data file.
112 lines
3.8 KiB
Go
112 lines
3.8 KiB
Go
package iceberg
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/apache/iceberg-go"
|
|
"github.com/google/uuid"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
)
|
|
|
|
func TestValidateCreateTableRequestRequiresName(t *testing.T) {
|
|
err := validateCreateTableRequest(CreateTableRequest{})
|
|
if !errors.Is(err, errTableNameRequired) {
|
|
t.Fatalf("validateCreateTableRequest() error = %v, want errTableNameRequired", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateCreateTableRequestAcceptsWithName(t *testing.T) {
|
|
err := validateCreateTableRequest(CreateTableRequest{Name: "orders"})
|
|
if err != nil {
|
|
t.Fatalf("validateCreateTableRequest() error = %v, want nil", err)
|
|
}
|
|
}
|
|
|
|
func TestIsStageCreateEnabledDefaultsToTrue(t *testing.T) {
|
|
t.Setenv("ICEBERG_ENABLE_STAGE_CREATE", "")
|
|
if !isStageCreateEnabled() {
|
|
t.Fatalf("isStageCreateEnabled() = false, want true")
|
|
}
|
|
}
|
|
|
|
func TestIsStageCreateEnabledFalseValues(t *testing.T) {
|
|
falseValues := []string{"0", "false", "FALSE", "no", "off"}
|
|
for _, value := range falseValues {
|
|
t.Setenv("ICEBERG_ENABLE_STAGE_CREATE", value)
|
|
if isStageCreateEnabled() {
|
|
t.Fatalf("isStageCreateEnabled() = true for value %q, want false", value)
|
|
}
|
|
}
|
|
}
|
|
|
|
func mustParseSchema(t *testing.T, raw string) *iceberg.Schema {
|
|
t.Helper()
|
|
var schema iceberg.Schema
|
|
if err := json.Unmarshal([]byte(raw), &schema); err != nil {
|
|
t.Fatalf("parse schema: %v", err)
|
|
}
|
|
return &schema
|
|
}
|
|
|
|
const variantSchema = `{"type":"struct","schema-id":0,"fields":[
|
|
{"id":1,"name":"id","required":true,"type":"long"},
|
|
{"id":2,"name":"payload","required":false,"type":"variant"}]}`
|
|
|
|
// A v3-only column type without format-version 3 is the client's mistake, and
|
|
// the reason has to travel back to them rather than only into the server log.
|
|
func TestNewTableMetadataRejectsV3TypeBelowV3(t *testing.T) {
|
|
_, err := newTableMetadata(uuid.New(), "s3://bkt/ns/t", mustParseSchema(t, variantSchema), nil, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("newTableMetadata() error = nil, want invalid schema")
|
|
}
|
|
if !errors.Is(err, iceberg.ErrInvalidSchema) {
|
|
t.Fatalf("newTableMetadata() error = %v, want ErrInvalidSchema", err)
|
|
}
|
|
|
|
// writeManagerError turns this into a 400; see TestWriteManagerError.
|
|
if !strings.Contains(err.Error(), "variant is not supported until v3") {
|
|
t.Errorf("error = %q, want the underlying reason", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestNewTableMetadataAcceptsV3TypeAtV3(t *testing.T) {
|
|
metadata, err := newTableMetadata(uuid.New(), "s3://bkt/ns/t", mustParseSchema(t, variantSchema), nil, nil,
|
|
iceberg.Properties{"format-version": "3"})
|
|
if err != nil {
|
|
t.Fatalf("newTableMetadata() error = %v, want nil", err)
|
|
}
|
|
if got := metadata.Version(); got != 3 {
|
|
t.Fatalf("metadata.Version() = %d, want 3", got)
|
|
}
|
|
if _, found := metadata.CurrentSchema().FindFieldByName("payload"); !found {
|
|
t.Error("variant field missing from stored schema")
|
|
}
|
|
}
|
|
|
|
// A LoadTable response must never carry nil metadata: it serializes as
|
|
// "metadata":null under HTTP 200, which no Iceberg client can parse.
|
|
func TestBuildLoadTableResultNeverReturnsNilMetadata(t *testing.T) {
|
|
cases := map[string]s3tables.GetTableResponse{
|
|
"no stored metadata": {MetadataLocation: "s3://bkt/ns/t/metadata/v1.metadata.json"},
|
|
"empty full metadata": {Metadata: &s3tables.TableMetadata{}},
|
|
"unparseable metadata": {Metadata: &s3tables.TableMetadata{FullMetadata: json.RawMessage(`{"nope":`)}},
|
|
}
|
|
for name, getResp := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
|
result, err := (&Server{}).buildLoadTableResult(r, getResp, "bkt", []string{"ns"}, "t")
|
|
if err != nil {
|
|
t.Fatalf("buildLoadTableResult() error = %v, want nil", err)
|
|
}
|
|
if result.Metadata == nil {
|
|
t.Fatal("buildLoadTableResult() returned nil metadata with no error")
|
|
}
|
|
})
|
|
}
|
|
}
|