Files
seaweedfs/weed/s3api/iceberg/manifest_repair.go
T
Nguyễn Lộc Phúc 195f8156de fix(iceberg): keep a losing table create from deleting the winner's metadata (#12474)
* iceberg: keep a losing table create from deleting the winner's metadata

* iceberg: keep a staged file whose published record was renamed away

A create answers a conflict when its record was already visible and
another request changed it before the create could back it out. The
cleanup then looked only at the original name, so a record renamed in
between read as absent and the file it still points at was deleted.

S3 Tables now reports that case with a conflict of its own, and both
create paths keep the staged file on it instead of probing the name.
A conflict over a still-hidden record, lost to another create, still
removes the file.

* iceberg: discard a losing create's metadata only while it is still its own

The cleanup deleted the staged file by name. The path can be reused
before it runs: the table is dropped and purged, and another create
stages the same v1.metadata.json and registers it, which the losing
request then deleted.

Staged files now carry a per-request token in an internal extended
attribute, and the cleanup deletes with that token as the filer's
ExpectedExtended precondition, checked under the path lock the delete
runs under. A file that took the path since is left in place.

* iceberg: classify a typed S3 Tables error by its code, not its message

isS3TablesAlreadyExists matched "already exists" in the message before
looking at the code, so an InternalError whose text mentioned an
existing entry read as the name being taken: CreateTable answered 409
instead of 500, and registrationRefused discarded a staged file the
catalog may still point at. A typed error now decides by its code; only
an untyped one is matched by message.

* iceberg: pin stage-create on in the stage-create conflict test

Stage-create is on by default, but a shell that sets
ICEBERG_ENABLE_STAGE_CREATE=false turned the expected 409 into a 501.
Setting it in the test keeps the result independent of the environment.

* iceberg: test two creates overlapping at the canonical metadata name

The concurrent-creators test only lines the creators up before they
start, so whether two of them ever collide at v1.metadata.json is up to
the scheduler. The new test fixes the order: the second create runs
while the first has staged v1.metadata.json and not yet registered it.
It checks that every staging attempt is exclusive, that the second falls
back to v1-<uuid>, that both files hold their own metadata while both
are staged, and that the loser removes only its own file.

* s3tables: a create whose unmark write never went out stayed hidden

confirmChild counted any failure to take a create's mark off as possibly
published, the lookup that precedes the write included. That lookup
failing leaves the mark on, so the record stayed hidden; if another
create then adopted it, the back-out reported a published record that
changed, and the Iceberg catalog kept a staged file no record names.

The mark-removal calls now go through a wrapper that notes whether an
UpdateEntry went out without the filer refusing it. Only then may the
record have been seen. A refused compare, a missing entry, or a failure
before any write leaves the create's record hidden.

* s3tables: pass mayBeVisible into backOutChild

Fixes the build after rebasing onto the backOutChild split.
2026-11-07 10:54:04 -08:00

619 lines
21 KiB
Go

package iceberg
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/linkedin/goavro/v2"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
// Some engines (ClickHouse's experimental Iceberg writes among them) commit
// snapshots whose manifest avro files omit the spec's field-id annotations and
// reference files by bucket-relative path. Both make the snapshot unreadable
// for other engines even though the data itself is fine. Since the catalog is
// the commit chokepoint, add-snapshot updates are repaired here: the manifest
// list and manifests are transcoded with the annotations added and the paths
// absolutized, and the snapshot is pointed at the repaired copies.
// avroFieldIDs describes the spec-mandated ids for one avro field and the
// fields nested beneath it (through unions, arrays, and records).
type avroFieldIDs struct {
id int
elementID int
children map[string]avroFieldIDs
}
// manifestListFieldIDs returns the fixed v2 manifest_file schema ids.
func manifestListFieldIDs() map[string]avroFieldIDs {
return map[string]avroFieldIDs{
"manifest_path": {id: 500},
"manifest_length": {id: 501},
"partition_spec_id": {id: 502},
"content": {id: 517},
"sequence_number": {id: 515},
"min_sequence_number": {id: 516},
"added_snapshot_id": {id: 503},
"added_files_count": {id: 504},
"existing_files_count": {id: 505},
"deleted_files_count": {id: 506},
"added_rows_count": {id: 512},
"existing_rows_count": {id: 513},
"deleted_rows_count": {id: 514},
"partitions": {id: 507, elementID: 508, children: map[string]avroFieldIDs{
"contains_null": {id: 509},
"contains_nan": {id: 518},
"lower_bound": {id: 510},
"upper_bound": {id: 511},
}},
"key_metadata": {id: 519},
"first_row_id": {id: 520},
}
}
// manifestEntryFieldIDs returns the fixed v2 manifest_entry schema ids. The
// partition struct is table-specific; its ids come from the partition-spec
// JSON embedded in the manifest's OCF metadata.
func manifestEntryFieldIDs(partitionIDs map[string]avroFieldIDs) map[string]avroFieldIDs {
return map[string]avroFieldIDs{
"status": {id: 0},
"snapshot_id": {id: 1},
"sequence_number": {id: 3},
"file_sequence_number": {id: 4},
"data_file": {id: 2, children: map[string]avroFieldIDs{
"content": {id: 134},
"file_path": {id: 100},
"file_format": {id: 101},
"partition": {id: 102, children: partitionIDs},
"record_count": {id: 103},
"file_size_in_bytes": {id: 104},
"column_sizes": {id: 108, children: kvFieldIDs(117, 118)},
"value_counts": {id: 109, children: kvFieldIDs(119, 120)},
"null_value_counts": {id: 110, children: kvFieldIDs(121, 122)},
"nan_value_counts": {id: 137, children: kvFieldIDs(138, 139)},
"lower_bounds": {id: 125, children: kvFieldIDs(126, 127)},
"upper_bounds": {id: 128, children: kvFieldIDs(129, 130)},
"key_metadata": {id: 131},
"split_offsets": {id: 132, elementID: 133},
"equality_ids": {id: 135, elementID: 136},
"sort_order_id": {id: 140},
"first_row_id": {id: 142},
"referenced_data_file": {id: 143},
"content_offset": {id: 144},
"content_size_in_bytes": {id: 145},
}},
}
}
func kvFieldIDs(keyID, valueID int) map[string]avroFieldIDs {
return map[string]avroFieldIDs{
"key": {id: keyID},
"value": {id: valueID},
}
}
// partitionFieldIDsFromSpec extracts name -> field-id from the partition-spec
// JSON carried in a manifest's OCF metadata.
func partitionFieldIDsFromSpec(specJSON []byte) (map[string]avroFieldIDs, error) {
if len(specJSON) == 0 {
return map[string]avroFieldIDs{}, nil
}
var fields []struct {
Name string `json:"name"`
FieldID int `json:"field-id"`
}
if err := json.Unmarshal(specJSON, &fields); err != nil {
return nil, fmt.Errorf("parse partition-spec metadata: %w", err)
}
ids := make(map[string]avroFieldIDs, len(fields))
for _, f := range fields {
ids[f.Name] = avroFieldIDs{id: f.FieldID}
}
return ids, nil
}
// annotateAvroRecordFields adds "field-id" to every field of a parsed avro
// record schema, recursing into nested types. Returns whether anything was
// added, and ok=false when a field has no known id (the schema is left for an
// unknown writer dialect rather than half-annotated).
func annotateAvroRecordFields(record map[string]any, ids map[string]avroFieldIDs) (changed, ok bool) {
fields, _ := record["fields"].([]any)
if len(fields) == 0 {
return false, true
}
for _, f := range fields {
field, isMap := f.(map[string]any)
if !isMap {
return changed, false
}
name, _ := field["name"].(string)
spec, known := ids[name]
if !known {
return changed, false
}
if _, has := field["field-id"]; !has {
field["field-id"] = spec.id
changed = true
}
typeChanged, typeOK := annotateAvroType(field["type"], spec)
changed = changed || typeChanged
if !typeOK {
return changed, false
}
}
return changed, true
}
// annotateAvroType annotates the schema node of a single field: unions are
// followed through their non-null branches, arrays get "element-id", and
// nested records recurse with the field's child ids.
func annotateAvroType(node any, spec avroFieldIDs) (changed, ok bool) {
switch t := node.(type) {
case []any: // union
for _, branch := range t {
branchChanged, branchOK := annotateAvroType(branch, spec)
changed = changed || branchChanged
if !branchOK {
return changed, false
}
}
return changed, true
case map[string]any:
switch t["type"] {
case "record":
childIDs := spec.children
if childIDs == nil {
childIDs = map[string]avroFieldIDs{}
}
return annotateAvroRecordFields(t, childIDs)
case "array":
if spec.elementID != 0 {
if _, has := t["element-id"]; !has {
t["element-id"] = spec.elementID
changed = true
}
}
itemChanged, itemOK := annotateAvroType(t["items"], spec)
return changed || itemChanged, itemOK
case "map":
// The spec wants key-id/value-id on genuine avro maps; this repair
// only knows the array-of-key_value encoding, so leave the file to
// its writer rather than half-annotating it.
return changed, false
default:
return false, true
}
default: // primitive name or named-type reference
return false, true
}
}
// transcodeManifestAvro decodes an avro OCF file, annotates its schema with
// the given ids, applies mutate to each datum and fixMeta to the OCF metadata,
// and re-encodes. When nothing changes, the original bytes are returned
// unchanged.
func transcodeManifestAvro(raw []byte, ids map[string]avroFieldIDs, mutate func(map[string]any) bool, fixMeta func(map[string][]byte) bool) ([]byte, bool, error) {
reader, err := goavro.NewOCFReader(bytes.NewReader(raw))
if err != nil {
return nil, false, fmt.Errorf("open avro: %w", err)
}
meta := reader.MetaData()
outMeta := make(map[string][]byte, len(meta))
for k, v := range meta {
if k == "avro.schema" || k == "avro.codec" {
continue
}
outMeta[k] = v
}
metaChanged := false
if fixMeta != nil {
metaChanged = fixMeta(outMeta)
}
var schema map[string]any
if err := json.Unmarshal(meta["avro.schema"], &schema); err != nil {
return nil, false, fmt.Errorf("parse avro schema: %w", err)
}
schemaChanged, ok := annotateAvroRecordFields(schema, ids)
if !ok {
return nil, false, fmt.Errorf("avro schema has fields outside the Iceberg manifest spec")
}
datumChanged := false
var datums []any
for reader.Scan() {
datum, err := reader.Read()
if err != nil {
return nil, false, fmt.Errorf("read avro datum: %w", err)
}
if record, isMap := datum.(map[string]any); isMap && mutate != nil {
if mutate(record) {
datumChanged = true
}
}
datums = append(datums, datum)
}
if err := reader.Err(); err != nil {
return nil, false, fmt.Errorf("scan avro: %w", err)
}
if !schemaChanged && !datumChanged && !metaChanged {
return raw, false, nil
}
annotatedSchema, err := json.Marshal(schema)
if err != nil {
return nil, false, err
}
var buf bytes.Buffer
writer, err := goavro.NewOCFWriter(goavro.OCFConfig{
W: &buf,
Schema: string(annotatedSchema),
MetaData: outMeta,
})
if err != nil {
return nil, false, fmt.Errorf("create avro writer: %w", err)
}
if err := writer.Append(datums); err != nil {
return nil, false, fmt.Errorf("write avro datums: %w", err)
}
return buf.Bytes(), true, nil
}
// manifestStore abstracts reading and writing files in a table's metadata
// directory so the repair logic is testable without a filer.
type manifestStore interface {
loadFile(ctx context.Context, location string) ([]byte, error)
saveFile(ctx context.Context, location string, data []byte) error
}
// absolutizeLocation resolves a bucket-relative path against the bucket of the
// table location. Locations that already carry a scheme pass through.
func absolutizeLocation(location, tableLocation string) string {
if location == "" || strings.Contains(location, "://") {
return location
}
bucket, _, err := parseS3Location(tableLocation)
if err != nil {
return location
}
return "s3://" + bucket + "/" + strings.TrimPrefix(location, "/")
}
// metadataFileName returns the file name when location is a direct child of
// the table's metadata directory, which is the only place repair will read
// from or write to.
func metadataFileName(tableLocation, location string) (string, bool) {
prefix := strings.TrimSuffix(tableLocation, "/") + "/metadata/"
name := strings.TrimPrefix(location, prefix)
if name == location || name == "" || strings.Contains(name, "/") {
return "", false
}
return name, true
}
const repairedManifestPrefix = "repaired-"
// maxRepairableManifestSize bounds how much manifest data the repair path
// buffers in memory; anything larger fails the repair and the commit
// proceeds with the writer's original files.
const maxRepairableManifestSize = 64 << 20
// hasFieldIDAnnotations is the cheap compliance probe: spec-compliant writers
// annotate every manifest avro schema with field-id, and a writer that
// annotates the manifest list annotates its manifests too.
func hasFieldIDAnnotations(raw []byte) bool {
reader, err := goavro.NewOCFReader(bytes.NewReader(raw))
if err != nil {
return false
}
return bytes.Contains(reader.MetaData()["avro.schema"], []byte(`"field-id"`))
}
// repairManifestList loads a snapshot's manifest list and, when it needs
// repair, rewrites it (and the manifests it references) into spec-compliant
// copies next to the originals. Returns the manifest list location the
// snapshot should reference.
func repairManifestList(ctx context.Context, store manifestStore, tableLocation, listLocation string) (string, bool, error) {
absListLocation := absolutizeLocation(listLocation, tableLocation)
listName, inMetadataDir := metadataFileName(tableLocation, absListLocation)
if !inMetadataDir {
return listLocation, false, nil
}
listBytes, err := store.loadFile(ctx, absListLocation)
if err != nil {
return listLocation, false, fmt.Errorf("load manifest list: %w", err)
}
if absListLocation == listLocation && hasFieldIDAnnotations(listBytes) {
return listLocation, false, nil
}
metadataDir := strings.TrimSuffix(tableLocation, "/") + "/metadata/"
var manifestErr error
repairedList, listChanged, err := transcodeManifestAvro(listBytes, manifestListFieldIDs(), func(record map[string]any) bool {
manifestPath, _ := record["manifest_path"].(string)
if manifestPath == "" || manifestErr != nil {
return false
}
absManifest := absolutizeLocation(manifestPath, tableLocation)
recordChanged := absManifest != manifestPath
if recordChanged {
record["manifest_path"] = absManifest
}
manifestName, ok := metadataFileName(tableLocation, absManifest)
if !ok {
return recordChanged
}
manifestBytes, err := store.loadFile(ctx, absManifest)
if err != nil {
manifestErr = fmt.Errorf("load manifest %s: %w", absManifest, err)
return recordChanged
}
repaired, changed, err := repairManifest(manifestBytes, tableLocation, manifestContentValue(record))
if err != nil {
manifestErr = fmt.Errorf("repair manifest %s: %w", absManifest, err)
return recordChanged
}
if !changed {
return recordChanged
}
repairedLocation := metadataDir + repairedManifestPrefix + manifestName
if err := store.saveFile(ctx, repairedLocation, repaired); err != nil {
manifestErr = fmt.Errorf("save repaired manifest %s: %w", repairedLocation, err)
return recordChanged
}
record["manifest_path"] = repairedLocation
record["manifest_length"] = int64(len(repaired))
return true
}, nil)
if err != nil {
return listLocation, false, err
}
if manifestErr != nil {
return listLocation, false, manifestErr
}
if !listChanged && absListLocation == listLocation {
return listLocation, false, nil
}
repairedListLocation := metadataDir + repairedManifestPrefix + listName
if listChanged {
if err := store.saveFile(ctx, repairedListLocation, repairedList); err != nil {
return listLocation, false, fmt.Errorf("save repaired manifest list: %w", err)
}
return repairedListLocation, true, nil
}
// Only the pointer was relative; the file itself is fine.
return absListLocation, true, nil
}
// manifestContentValue reads the manifest-list entry's content field
// (0 = data, 1 = deletes); v1 lists have no such field and default to data.
// Union-typed values arrive from goavro as a single-branch map.
func manifestContentValue(record map[string]any) int {
value := record["content"]
if union, isUnion := value.(map[string]any); isUnion {
for _, branch := range union {
value = branch
break
}
}
switch v := value.(type) {
case int32:
return int(v)
case int64:
return int(v)
case int:
return v
}
return 0
}
// fixManifestOCFMetadata fixes OCF metadata keys the spec requires of
// manifests but lax writers omit or contradict: "content" must agree with the
// manifest-list entry, which is what readers trust for scan planning, and
// "partition-spec-id" (ClickHouse writes the underscore variant).
func fixManifestOCFMetadata(listContent int) func(map[string][]byte) bool {
return func(meta map[string][]byte) bool {
changed := false
content := "data"
if listContent == 1 {
content = "deletes"
}
if string(meta["content"]) != content {
meta["content"] = []byte(content)
changed = true
}
if _, ok := meta["partition-spec-id"]; !ok {
if v, ok := meta["partition_spec_id"]; ok {
meta["partition-spec-id"] = v
changed = true
}
}
return changed
}
}
// repairManifest annotates a single manifest's schema and absolutizes the
// data file paths inside it. listContent is the content declared by the
// manifest-list entry referencing this manifest.
func repairManifest(raw []byte, tableLocation string, listContent int) ([]byte, bool, error) {
reader, err := goavro.NewOCFReader(bytes.NewReader(raw))
if err != nil {
return nil, false, fmt.Errorf("open manifest avro: %w", err)
}
partitionIDs, err := partitionFieldIDsFromSpec(reader.MetaData()["partition-spec"])
if err != nil {
return nil, false, err
}
return transcodeManifestAvro(raw, manifestEntryFieldIDs(partitionIDs), func(record map[string]any) bool {
dataFile, _ := record["data_file"].(map[string]any)
if dataFile == nil {
return false
}
filePath, _ := dataFile["file_path"].(string)
absPath := absolutizeLocation(filePath, tableLocation)
if absPath == filePath {
return false
}
dataFile["file_path"] = absPath
return true
}, fixManifestOCFMetadata(listContent))
}
// repairAddSnapshotUpdates rewrites the manifest-list references of
// add-snapshot updates whose manifests need repair. Repair is best effort: on
// any error the original update is kept and the commit proceeds as it would
// have without repair.
func repairAddSnapshotUpdates(ctx context.Context, store manifestStore, tableLocation string, rawUpdates []json.RawMessage) ([]json.RawMessage, bool) {
changedAny := false
out := make([]json.RawMessage, len(rawUpdates))
copy(out, rawUpdates)
for i, rawUpdate := range rawUpdates {
var probe struct {
Action string `json:"action"`
}
if err := json.Unmarshal(rawUpdate, &probe); err != nil || probe.Action != "add-snapshot" {
continue
}
var update map[string]json.RawMessage
if err := json.Unmarshal(rawUpdate, &update); err != nil {
continue
}
var snapshot map[string]json.RawMessage
if err := json.Unmarshal(update["snapshot"], &snapshot); err != nil {
continue
}
var listLocation string
if err := json.Unmarshal(snapshot["manifest-list"], &listLocation); err != nil || listLocation == "" {
continue
}
repairedLocation, changed, err := repairManifestList(ctx, store, tableLocation, listLocation)
if err != nil {
glog.Warningf("Iceberg: manifest repair for %s skipped: %v", listLocation, err)
continue
}
if !changed {
continue
}
locationJSON, err := json.Marshal(repairedLocation)
if err != nil {
continue
}
snapshot["manifest-list"] = locationJSON
snapshotJSON, err := json.Marshal(snapshot)
if err != nil {
continue
}
update["snapshot"] = snapshotJSON
updateJSON, err := json.Marshal(update)
if err != nil {
continue
}
out[i] = updateJSON
changedAny = true
glog.V(1).Infof("Iceberg: repaired manifest list %s -> %s", listLocation, repairedLocation)
}
return out, changedAny
}
// serverManifestStore reads and writes table metadata files through the filer.
type serverManifestStore struct {
server *Server
tableLocation string
}
func (st *serverManifestStore) split(location string) (bucket, tablePath, fileName string, err error) {
fileName, ok := metadataFileName(st.tableLocation, location)
if !ok {
return "", "", "", fmt.Errorf("%s is outside the table metadata directory", location)
}
bucket, tablePath, err = parseS3Location(st.tableLocation)
return bucket, tablePath, fileName, err
}
func (st *serverManifestStore) loadFile(ctx context.Context, location string) ([]byte, error) {
bucket, tablePath, fileName, err := st.split(location)
if err != nil {
return nil, err
}
return st.server.loadTableMetadataBlob(ctx, bucket, tablePath, fileName)
}
func (st *serverManifestStore) saveFile(ctx context.Context, location string, data []byte) error {
bucket, tablePath, fileName, err := st.split(location)
if err != nil {
return err
}
return st.server.saveMetadataBlob(ctx, bucket, tablePath, fileName, data, "application/avro", false, nil)
}
// repairAddSnapshotManifests is the server entry point used by the commit
// handlers.
func (s *Server) repairAddSnapshotManifests(ctx context.Context, tableLocation string, rawUpdates []json.RawMessage) ([]json.RawMessage, bool) {
store := &serverManifestStore{server: s, tableLocation: tableLocation}
return repairAddSnapshotUpdates(ctx, store, tableLocation, rawUpdates)
}
// lookupFileIDAdapter lets filer.StreamContent resolve volume locations
// through the iceberg server's filer client.
type lookupFileIDAdapter struct {
fn wdclient.LookupFileIdFunctionType
}
func (a *lookupFileIDAdapter) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
return a.fn
}
// loadTableMetadataBlob reads a file from the table's metadata directory,
// following chunks for files that were written through the S3 gateway rather
// than inline by the catalog itself.
func (s *Server) loadTableMetadataBlob(ctx context.Context, bucketName, tablePath, fileName string) ([]byte, error) {
var entry *filer_pb.Entry
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: metadataDirPath(bucketName, tablePath),
Name: fileName,
})
if err != nil {
return err
}
entry = resp.Entry
return nil
})
if err != nil {
return nil, err
}
if entry == nil {
return nil, fmt.Errorf("no entry for %s", fileName)
}
if len(entry.Content) > 0 || len(entry.GetChunks()) == 0 {
return entry.Content, nil
}
if size := filer.FileSize(entry); size > maxRepairableManifestSize {
return nil, fmt.Errorf("%s is %d bytes, larger than the %d byte repair limit", fileName, size, maxRepairableManifestSize)
}
fullClient, ok := s.filerClient.(filer_pb.FilerClient)
if !ok {
return nil, fmt.Errorf("filer client cannot resolve chunk locations")
}
var buf bytes.Buffer
lookup := &lookupFileIDAdapter{fn: filer.LookupFn(fullClient)}
if err := filer.StreamContent(lookup, &buf, entry.GetChunks(), 0, int64(filer.FileSize(entry))); err != nil {
return nil, fmt.Errorf("read %s: %w", fileName, err)
}
return buf.Bytes(), nil
}