mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-20 13:04:25 +02:00
s3tables: accept .puffin deletion vectors in table data/ (#11844)
* s3tables: accept .puffin data files for Iceberg v3 deletion vectors Iceberg v3 writes deletion vectors as Puffin files under a table's data/ directory, so merge-on-read DELETE and UPDATE on v3 tables were refused with AccessDenied by the table-bucket upload validator. Fixes #11842 * s3tables: name all accepted data formats in rejection error The message stopped at .avro even though .lance and now .puffin are accepted. * s3tables: cover deletion-vector filenames in data layout test Uses the names real engines emit: Spark/Iceberg Java {seq}-{task}-{uuid}-{counter}-deletes.puffin, UUID-suffixed names, and the same under a partition directory.
This commit is contained in:
1 parent
ce93660abb
commit
117ea58a83
2 files changed
+6
-2
No files matched your search
@@ -15,7 +15,7 @@ import (
|
||||
// - metadata/v*.metadata.json (table metadata)
|
||||
// - metadata/snap-*.avro (snapshot manifest lists)
|
||||
// - metadata/*.avro (manifest files)
|
||||
// - data/*.parquet, data/*.orc, data/*.avro (data files)
|
||||
// - data/*.parquet, *.orc, *.avro, *.puffin (data files)
|
||||
|
||||
const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
|
||||
|
||||
@@ -75,6 +75,7 @@ var (
|
||||
regexp.MustCompile(`^[^/]+\.parquet$`), // Parquet files
|
||||
regexp.MustCompile(`^[^/]+\.orc$`), // ORC files
|
||||
regexp.MustCompile(`^[^/]+\.avro$`), // Avro files
|
||||
regexp.MustCompile(`^[^/]+\.puffin$`), // Iceberg v3 deletion vectors
|
||||
regexp.MustCompile(`^[^/]+\.lance$`), // Lance fragments
|
||||
}
|
||||
|
||||
@@ -209,7 +210,7 @@ func validateFilePatterns(filename string, isMetadata bool) error {
|
||||
errorMsg = "invalid metadata file format: must be a valid Iceberg metadata, manifest, or snapshot file"
|
||||
} else {
|
||||
patterns = dataFilePatterns
|
||||
errorMsg = "invalid data file format: must be .parquet, .orc, or .avro"
|
||||
errorMsg = "invalid data file format: must be .parquet, .orc, .avro, .puffin or .lance"
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
|
||||
@@ -90,6 +90,9 @@ func TestIcebergLayoutValidator_AcceptsRealWorldDataFiles(t *testing.T) {
|
||||
"data/part-00000.parquet",
|
||||
"data/some-file.orc",
|
||||
"data/year=2026/month=05/00000-0-uuid.parquet",
|
||||
"data/00000-2-c56b1e21-828c-40ad-8f7b-739c6cb82dcf-00001-deletes.puffin",
|
||||
"data/82e3eec4-3aee-414f-a444-94c03c641d20-deletes.puffin",
|
||||
"data/year=2026/month=05/00000-2-c56b1e21-828c-40ad-8f7b-739c6cb82dcf-00001-deletes.puffin",
|
||||
}
|
||||
v := NewIcebergLayoutValidator()
|
||||
for _, p := range cases {
|
||||
|
||||
Reference in new issue
Block a user