From 117ea58a83267a37dbf1fd0c433253c38e5a40eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 19 Oct 2026 16:59:08 -0700 Subject: [PATCH] s3tables: accept .puffin deletion vectors in table data/ (#11844) * s3tables: accept .puffin data files for Iceberg v3 deletion vectors Iceberg v3 writes deletion vectors as Puffin files under a table's data/ directory, so merge-on-read DELETE and UPDATE on v3 tables were refused with AccessDenied by the table-bucket upload validator. Fixes #11842 * s3tables: name all accepted data formats in rejection error The message stopped at .avro even though .lance and now .puffin are accepted. * s3tables: cover deletion-vector filenames in data layout test Uses the names real engines emit: Spark/Iceberg Java {seq}-{task}-{uuid}-{counter}-deletes.puffin, UUID-suffixed names, and the same under a partition directory. --- weed/s3api/s3tables/iceberg_layout.go | 5 +++-- weed/s3api/s3tables/iceberg_layout_test.go | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3tables/iceberg_layout.go b/weed/s3api/s3tables/iceberg_layout.go index f28ddd813a..00c2030285 100644 --- a/weed/s3api/s3tables/iceberg_layout.go +++ b/weed/s3api/s3tables/iceberg_layout.go @@ -15,7 +15,7 @@ import ( // - metadata/v*.metadata.json (table metadata) // - metadata/snap-*.avro (snapshot manifest lists) // - metadata/*.avro (manifest files) -// - data/*.parquet, data/*.orc, data/*.avro (data files) +// - data/*.parquet, *.orc, *.avro, *.puffin (data files) const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}` @@ -75,6 +75,7 @@ var ( regexp.MustCompile(`^[^/]+\.parquet$`), // Parquet files regexp.MustCompile(`^[^/]+\.orc$`), // ORC files regexp.MustCompile(`^[^/]+\.avro$`), // Avro files + regexp.MustCompile(`^[^/]+\.puffin$`), // Iceberg v3 deletion vectors regexp.MustCompile(`^[^/]+\.lance$`), // Lance fragments } @@ -209,7 +210,7 @@ func validateFilePatterns(filename string, isMetadata bool) error { errorMsg = "invalid metadata file format: must be a valid Iceberg metadata, manifest, or snapshot file" } else { patterns = dataFilePatterns - errorMsg = "invalid data file format: must be .parquet, .orc, or .avro" + errorMsg = "invalid data file format: must be .parquet, .orc, .avro, .puffin or .lance" } for _, pattern := range patterns { diff --git a/weed/s3api/s3tables/iceberg_layout_test.go b/weed/s3api/s3tables/iceberg_layout_test.go index 701be3532b..855361297f 100644 --- a/weed/s3api/s3tables/iceberg_layout_test.go +++ b/weed/s3api/s3tables/iceberg_layout_test.go @@ -90,6 +90,9 @@ func TestIcebergLayoutValidator_AcceptsRealWorldDataFiles(t *testing.T) { "data/part-00000.parquet", "data/some-file.orc", "data/year=2026/month=05/00000-0-uuid.parquet", + "data/00000-2-c56b1e21-828c-40ad-8f7b-739c6cb82dcf-00001-deletes.puffin", + "data/82e3eec4-3aee-414f-a444-94c03c641d20-deletes.puffin", + "data/year=2026/month=05/00000-2-c56b1e21-828c-40ad-8f7b-739c6cb82dcf-00001-deletes.puffin", } v := NewIcebergLayoutValidator() for _, p := range cases {