mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
fix(iceberg): serve a registered table's metadata on load and commit (#12159)
* iceberg: serve a registered table's metadata on load and commit RegisterTable saved only the metadata location, and loads and commits never read the file there. They served a placeholder with no schema, no snapshots and a new random table UUID on every load, so a registered table read as empty, an INSERT failed against the empty schema, and every other commit failed its assert-table-uuid requirement. Metadata uploaded through the S3 gateway, as a migrated table's is, is stored in chunks that registration did not read, so it registered an empty table. Read the file following its chunks, reject one that is not Iceberg metadata with a 400, and keep its content and table UUID in the record the way CreateTable does. Records registered before this change hold only the location. Loads and commits now read the file there, and the first commit stores it in the record under the file's table UUID. * iceberg: transactions read a location-only registration's metadata file * iceberg: cover transactions on a location-only registration * iceberg: refresh the default name mapping on transaction commits * iceberg: cover name-mapping refresh in transactions * iceberg: roll a transaction back to a registration's metadata file A transaction that failed after flipping a table registered before the record kept its metadata could not undo the flip. The rollback restored the record's prior metadata, which for such a table is only a location, and UpdateTable keeps the full metadata a flip stored when the restore sends none. Loads went on serving the change the client was told had failed, and a client retrying the transaction could apply it twice. Restore the metadata loads served from the file instead, under the file's table UUID. * iceberg: register oversized metadata by its location Registration sends the metadata file's content to the S3 Tables manager, which refuses a request over its 10MB body limit, so a table whose metadata is larger could no longer be registered at all. Register such a file by its location and table UUID alone; loads read it from that location, as they do for a record that never kept its metadata. Export the limit as s3tables.MaxRequestBodySize so the catalog can check against it. A commit to such a table still sends the whole metadata to UpdateTable and stays over the limit. * iceberg: commit oversized metadata by its location * iceberg: let maintenance follow a location-only table record * admin: read a table's metadata file when the record keeps none * iceberg: bound the metadata file a location-only record reads * iceberg: bound only chunked metadata reads in the worker Inline content is what the catalog itself wrote and the REST load path accepts it at any size, so rejecting it in loadMetadata stranded tables over 64 MiB from every maintenance scan. Cap only the chunked read, where the recorded location can name any object in the bucket. --------- Co-authored-by: chrislusf <chrislusf@users.noreply.github.com>
This commit is contained in:
1 parent
a816c6202c
commit
b62bbb99fc
20 files changed
+1123
-37
No files matched your search
@@ -0,0 +1,117 @@
|
||||
package catalog_spark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// TestSparkRegisterTableServesItsData registers a table under a new name from
|
||||
// the current metadata file of another and checks that Spark reads the
|
||||
// registered rows and can write on top of them. One file is the one the
|
||||
// catalog wrote; the other is a copy uploaded through the S3 gateway, the way
|
||||
// the metadata of a table migrated in arrives, which the filer stores in
|
||||
// chunks rather than inline. A registered table used to load as an empty
|
||||
// placeholder: SELECT found no rows and INSERT failed its table UUID check.
|
||||
func TestSparkRegisterTableServesItsData(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env, _, tableBucket := setupSparkTestEnv(t)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
upload bool
|
||||
}{
|
||||
{"metadata file written by the catalog", false},
|
||||
{"metadata file uploaded through S3", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
namespace := "register_" + randomString(6)
|
||||
source := fmt.Sprintf("iceberg.%s.src_%s", namespace, randomString(6))
|
||||
table := "reg_" + randomString(6)
|
||||
ident := fmt.Sprintf("iceberg.%s.%s", namespace, table)
|
||||
|
||||
output := runSparkPySQL(t, env.sparkContainer, fmt.Sprintf(`
|
||||
spark.sql("CREATE NAMESPACE iceberg.%[1]s")
|
||||
spark.sql("CREATE TABLE %[2]s (id INT, name STRING) USING iceberg")
|
||||
spark.sql("INSERT INTO %[2]s VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')")
|
||||
meta = spark.sql("SELECT file FROM %[2]s.metadata_log_entries ORDER BY timestamp DESC LIMIT 1").collect()[0]['file']
|
||||
print(f"metadata file: {meta}")
|
||||
`, namespace, source), env.icebergRestPort, env.s3Port)
|
||||
match := regexp.MustCompile(`metadata file: (\S+)`).FindStringSubmatch(output)
|
||||
if match == nil {
|
||||
t.Fatalf("create/insert of %s failed, output:\n%s", source, output)
|
||||
}
|
||||
metadataFile := match[1]
|
||||
if tc.upload {
|
||||
metadataFile = uploadMetadataCopy(t, env, tableBucket, metadataFile)
|
||||
}
|
||||
|
||||
output = runSparkPySQL(t, env.sparkContainer, fmt.Sprintf(`
|
||||
spark.sql("CALL iceberg.system.register_table('%[1]s.%[2]s', '%[3]s')")
|
||||
registered = spark.sql("SELECT COUNT(*) AS c FROM %[4]s").collect()[0]['c']
|
||||
print(f"registered row count: {registered}")
|
||||
spark.sql("INSERT INTO %[4]s VALUES (4, 'delta')")
|
||||
inserted = spark.sql("SELECT COUNT(*) AS c FROM %[4]s").collect()[0]['c']
|
||||
print(f"row count after insert: {inserted}")
|
||||
`, namespace, table, metadataFile, ident), env.icebergRestPort, env.s3Port)
|
||||
if !strings.Contains(output, "registered row count: 3") {
|
||||
t.Fatalf("the table registered from %s does not serve its rows, output:\n%s", metadataFile, output)
|
||||
}
|
||||
if !strings.Contains(output, "row count after insert: 4") {
|
||||
t.Fatalf("INSERT into the table registered from %s failed, output:\n%s", metadataFile, output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// uploadMetadataCopy puts a copy of metadataFile next to it through the S3
|
||||
// gateway and returns the copy's location.
|
||||
func uploadMetadataCopy(t *testing.T, env *TestEnvironment, bucket, metadataFile string) string {
|
||||
t.Helper()
|
||||
key, ok := strings.CutPrefix(metadataFile, "s3://"+bucket+"/")
|
||||
if !ok {
|
||||
t.Fatalf("metadata file %s is not in bucket %s", metadataFile, bucket)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cfg := aws.Config{
|
||||
Region: "us-east-1",
|
||||
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
|
||||
BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
|
||||
}
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
|
||||
|
||||
obj, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
if err != nil {
|
||||
t.Fatalf("GetObject %s: %v", key, err)
|
||||
}
|
||||
content, err := io.ReadAll(obj.Body)
|
||||
obj.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", key, err)
|
||||
}
|
||||
|
||||
copyKey := path.Join(path.Dir(key), "00003-uploaded.metadata.json")
|
||||
if _, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(copyKey),
|
||||
Body: bytes.NewReader(content),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutObject %s: %v", copyKey, err)
|
||||
}
|
||||
return "s3://" + bucket + "/" + copyKey
|
||||
}
|
||||
Reference in new issue
Block a user