mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 09:20:50 +01:00
Nguyễn Lộc Phúcandchrislusf
b62bbb99fc
* iceberg: serve a registered table's metadata on load and commit RegisterTable saved only the metadata location, and loads and commits never read the file there. They served a placeholder with no schema, no snapshots and a new random table UUID on every load, so a registered table read as empty, an INSERT failed against the empty schema, and every other commit failed its assert-table-uuid requirement. Metadata uploaded through the S3 gateway, as a migrated table's is, is stored in chunks that registration did not read, so it registered an empty table. Read the file following its chunks, reject one that is not Iceberg metadata with a 400, and keep its content and table UUID in the record the way CreateTable does. Records registered before this change hold only the location. Loads and commits now read the file there, and the first commit stores it in the record under the file's table UUID. * iceberg: transactions read a location-only registration's metadata file * iceberg: cover transactions on a location-only registration * iceberg: refresh the default name mapping on transaction commits * iceberg: cover name-mapping refresh in transactions * iceberg: roll a transaction back to a registration's metadata file A transaction that failed after flipping a table registered before the record kept its metadata could not undo the flip. The rollback restored the record's prior metadata, which for such a table is only a location, and UpdateTable keeps the full metadata a flip stored when the restore sends none. Loads went on serving the change the client was told had failed, and a client retrying the transaction could apply it twice. Restore the metadata loads served from the file instead, under the file's table UUID. * iceberg: register oversized metadata by its location Registration sends the metadata file's content to the S3 Tables manager, which refuses a request over its 10MB body limit, so a table whose metadata is larger could no longer be registered at all. Register such a file by its location and table UUID alone; loads read it from that location, as they do for a record that never kept its metadata. Export the limit as s3tables.MaxRequestBodySize so the catalog can check against it. A commit to such a table still sends the whole metadata to UpdateTable and stays over the limit. * iceberg: commit oversized metadata by its location * iceberg: let maintenance follow a location-only table record * admin: read a table's metadata file when the record keeps none * iceberg: bound the metadata file a location-only record reads * iceberg: bound only chunked metadata reads in the worker Inline content is what the catalog itself wrote and the REST load path accepts it at any size, so rejecting it in loadMetadata stranded tables over 64 MiB from every maintenance scan. Cap only the chunked read, where the recorded location can name any object in the bucket. --------- Co-authored-by: chrislusf <chrislusf@users.noreply.github.com>
80 lines
3.2 KiB
Go
80 lines
3.2 KiB
Go
package iceberg
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
)
|
|
|
|
func TestWantsVendedCredentials(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
values []string
|
|
want bool
|
|
}{
|
|
{name: "no header", want: false},
|
|
{name: "vended credentials", values: []string{"vended-credentials"}, want: true},
|
|
{name: "mechanism list", values: []string{"remote-signing,vended-credentials"}, want: true},
|
|
{name: "spaced and mixed case", values: []string{"Remote-Signing, Vended-Credentials"}, want: true},
|
|
{name: "repeated header", values: []string{"remote-signing", "vended-credentials"}, want: true},
|
|
{name: "remote signing only", values: []string{"remote-signing"}, want: false},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
|
for _, v := range tc.values {
|
|
r.Header.Add(accessDelegationHeader, v)
|
|
}
|
|
if got := wantsVendedCredentials(r); got != tc.want {
|
|
t.Fatalf("wantsVendedCredentials(%v) = %v, want %v", tc.values, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A client that asked for vended credentials replaces its own storage
|
|
// credentials with whatever the catalog returns. With no vending configured
|
|
// there are none, so an endpoint by itself would leave the client signing
|
|
// nothing and every data file read and write would come back 403.
|
|
func TestBuildFileIOConfigWithholdsEndpointFromCredentialVendingClients(t *testing.T) {
|
|
s := &Server{s3Endpoint: "http://seaweed.example:8333"}
|
|
|
|
r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
|
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
|
if got, _ := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil); len(got) != 0 {
|
|
t.Fatalf("buildFileIOConfig() = %v, want empty for a vended-credentials request", got)
|
|
}
|
|
|
|
plain := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
|
if got, _ := s.buildFileIOConfig(plain, "s3://warehouse/ns/t", nil); got["s3.endpoint"] != s.s3Endpoint {
|
|
t.Fatalf("s3.endpoint = %q, want %q", got["s3.endpoint"], s.s3Endpoint)
|
|
}
|
|
}
|
|
|
|
// The body of a load response depends on the delegation header, so a cache in
|
|
// front of the catalog must not key on the URL alone.
|
|
func TestWriteLoadResultVariesOnTheDelegationHeader(t *testing.T) {
|
|
rec := httptest.NewRecorder()
|
|
writeLoadResult(rec, http.StatusOK, LoadTableResult{})
|
|
if got := rec.Header().Get("Vary"); got != accessDelegationHeader {
|
|
t.Fatalf("Vary = %q, want %q", got, accessDelegationHeader)
|
|
}
|
|
}
|
|
|
|
func TestLoadTableResultOmitsConfigForCredentialVendingClients(t *testing.T) {
|
|
s := &Server{filerClient: newMemFiler(), s3Endpoint: "http://seaweed.example:8333"}
|
|
r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
|
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
|
|
|
getResp := s3tables.GetTableResponse{MetadataLocation: "s3://bkt/ns/t/metadata/v1.metadata.json"}
|
|
result, err := s.buildLoadTableResult(r, getResp, "bkt", []string{"ns"}, "t")
|
|
if err != nil {
|
|
t.Fatalf("buildLoadTableResult() error = %v", err)
|
|
}
|
|
if len(result.Config) != 0 {
|
|
t.Fatalf("LoadTableResult.Config = %v, want empty", result.Config)
|
|
}
|
|
}
|