mount: surface ENOSPC instead of endless waiting when the cluster is full (#10341)

When every volume is full, a chunk upload fails only after the assign
retry budget, the failed chunk's data is dropped, and the mount kept
accepting writes anyway. cp would crawl for hours pushing the rest of
the file through a pipeline that could not persist it, and only close()
reported an error - a generic EIO.

Poison the file handle on the first failed chunk upload so subsequent
writes fail immediately, and map "no writable volumes" / "no free
volumes" upload errors to ENOSPC so the writing process aborts with
"No space left on device". Also guard lastErr with a mutex: it was
written concurrently by uploader goroutines, and keep the first error
so later failures do not mask the root cause.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-07-15 22:01:19 -07:00
1 parent 25ab4c3cac
commit 5553e7b876
7 files changed
+117 -6

No files matched your search

+35
View File
@@ -44,6 +44,41 @@ func TestGrpcErrorToFuseStatusDropsCanceledThroughPercentV(t *testing.T) {
}
}
// A full cluster must surface as ENOSPC so cp/rsync abort with "No space left
// on device" instead of a generic I/O error. The master's message reaches the
// mount stringified inside AssignVolumeResponse.Error, in one of two shapes:
// the load-shedding gRPC status text or the topology's plain error text.
func TestWriteErrorToFuseStatus(t *testing.T) {
cases := []struct {
name string
err error
want fuse.Status
}{
{
"assign shed during growth",
errors.New("upload data: filerGrpcAddress assign volume: assign volume failure count:1: assign volume: rpc error: code = ResourceExhausted desc = no writable volumes for replication:000 collection:, volume growth in progress"),
fuse.Status(syscall.ENOSPC),
},
{
"no free volumes left",
errors.New("flush data: upload data: assign volume: No writable volumes available for collection: replication:000 ttl: and no free volumes left for {replication:000}"),
fuse.Status(syscall.ENOSPC),
},
{
"unrelated upload failure",
errors.New("upload data: put http://volume:8080/3,0144: connection reset by peer"),
fuse.EIO,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := writeErrorToFuseStatus(tc.err); got != tc.want {
t.Fatalf("writeErrorToFuseStatus(%v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}
func TestIsRetryableFilerError(t *testing.T) {
cases := []struct {
name string