test(fuse_failover): dump chunk list and hex on append corruption (#11285)

* test(fuse_failover): dump chunk list and hex on append corruption

The failover append test (TestAppendWhileVolumeServerRestarts) failed
in CI with an 8-byte NUL region at offset 632 that appeared in both
the writer mount and the filer own view, but the failure message
only showed a quoted-string window around the divergence. That is
not enough to tell which chunk covered the zeroed bytes or which
volume server held it, so the next recurrence would be just as
unattributable.

Add a FileChunkList helper that reads the filer resolved chunk
list, and on failure dump:
  - every chunk fid, offset, size, volume id, and current master
    holders, flagging the chunk that covers the first divergence;
  - a hex+ASCII dump of the writer mount around the divergence so
    the exact zero-filled region is visible byte-for-byte.

No production code is touched; this only makes the test fail louder.

* test(fuse_failover): preserve diagnostic collection errors

Address review feedback from CodeRabbit and Greptile on PR #11285:

- FileChunkList now returns the wrapped ParseUint error when
  fid.volume_id is zero and the file_id prefix is invalid, matching
  FileVolumeIds instead of silently keeping vid=0 (which would
  query /dir/lookup?volumeId=0 and report the wrong holders).
- dumpChunkList captures the VolumeHolders error and renders it as
  'lookup failed: ...' so a failed master request is distinguishable
  from a successful lookup with no holders (both previously showed
  'holders=[unknown]').
- runChaosAppend captures the writer-mount read error and includes
  it in the failure message so an unavailable writer view is not
  mistaken for corrupted content.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-09-11 23:08:39 -07:00
1 parent c46f82d29a
commit 3b4a681e53
2 files changed
+134 -4

No files matched your search

+51
View File
@@ -309,6 +309,57 @@ func (c *failoverCluster) FileVolumeIds(path string) ([]uint32, error) {
return vids, nil
}
// FileChunkList returns the filer's chunk list for a file: each chunk's file
// id, logical offset, size, and the volume id it lives on. Used to pinpoint
// which chunk covers a corrupted byte range and which volume server holds it.
func (c *failoverCluster) FileChunkList(path string) ([]struct {
FileId string
Offset int64
Size uint64
VolumeId uint32
}, error) {
body, err := c.FilerGet(path + "?metadata=true&resolveManifest=true")
if err != nil {
return nil, err
}
var entry struct {
Chunks []struct {
FileId string `json:"file_id"`
Offset int64 `json:"offset"`
Size uint64 `json:"size"`
Fid struct {
VolumeId uint32 `json:"volume_id"`
} `json:"fid"`
} `json:"chunks"`
}
if err = json.Unmarshal(body, &entry); err != nil {
return nil, fmt.Errorf("decode entry %s: %w", path, err)
}
out := make([]struct {
FileId string
Offset int64
Size uint64
VolumeId uint32
}, len(entry.Chunks))
for i, ch := range entry.Chunks {
vid := ch.Fid.VolumeId
if vid == 0 && ch.FileId != "" {
parsed, parseErr := strconv.ParseUint(strings.SplitN(ch.FileId, ",", 2)[0], 10, 32)
if parseErr != nil {
return nil, fmt.Errorf("parse file id %s: %w", ch.FileId, parseErr)
}
vid = uint32(parsed)
}
out[i] = struct {
FileId string
Offset int64
Size uint64
VolumeId uint32
}{ch.FileId, ch.Offset, ch.Size, vid}
}
return out, nil
}
// VolumeHolders returns the volume server addresses the master currently lists
// for a volume id.
func (c *failoverCluster) VolumeHolders(vid uint32) ([]string, error) {