shell: restore the tier download source when its readonly mark fails

VolumeMarkReadonly blocks local writes and deletes before its second master
notification, so a failed notification returns an error with the source
already frozen. Restore it whenever the status was read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 committed 2026-11-12 19:46:00 -08:00
1 parent fe248c88a2
commit 2e0ccfbd0f
2 files changed
+37 -4

No files matched your search

+10 -4
View File
@@ -190,10 +190,12 @@ func downloadAndReplicateFromRemoteTier(grpcDialOption grpc.DialOption, writer i
} }
restore, err := freezeTierDownloadSource(grpcDialOption, vid, source) restore, err := freezeTierDownloadSource(grpcDialOption, vid, source)
if restore != nil {
defer restore(writer)
}
if err != nil { if err != nil {
return fmt.Errorf("block deletes on volume %d at %s: %v", vid, source, err) return fmt.Errorf("block deletes on volume %d at %s: %v", vid, source, err)
} }
defer restore(writer)
// Copy over all remote-backed replicas but one, re-reading the topology // Copy over all remote-backed replicas but one, re-reading the topology
// so a replica that was offline or created meanwhile is replaced too. The // so a replica that was offline or created meanwhile is replaced too. The
@@ -245,18 +247,22 @@ func downloadAndReplicateFromRemoteTier(grpcDialOption grpc.DialOption, writer i
// freezeTierDownloadSource blocks deletes on the copy source: a delete applied // freezeTierDownloadSource blocks deletes on the copy source: a delete applied
// after VolumeCopy read the source index would be lost on the replaced replica. // after VolumeCopy read the source index would be lost on the replaced replica.
// restore brings back the writable mark, or reloads the persisted state. // restore brings back the writable mark, or reloads the persisted state. It is
// also returned when the mark fails, since the server may have applied it.
func freezeTierDownloadSource(grpcDialOption grpc.DialOption, vid needle.VolumeId, source pb.ServerAddress) (restore func(io.Writer), err error) { func freezeTierDownloadSource(grpcDialOption grpc.DialOption, vid needle.VolumeId, source pb.ServerAddress) (restore func(io.Writer), err error) {
var wasWritable bool var wasWritable, statusRead bool
err = operation.WithVolumeServerClient(false, source, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { err = operation.WithVolumeServerClient(false, source, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, statusErr := client.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{VolumeId: uint32(vid)}) resp, statusErr := client.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{VolumeId: uint32(vid)})
if statusErr != nil { if statusErr != nil {
return statusErr return statusErr
} }
wasWritable = !resp.IsReadOnly wasWritable, statusRead = !resp.IsReadOnly, true
_, markErr := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: uint32(vid)}) _, markErr := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: uint32(vid)})
return markErr return markErr
}) })
if !statusRead {
return nil, err
}
restore = func(writer io.Writer) { restore = func(writer io.Writer) {
restoreErr := operation.WithVolumeServerClient(false, source, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { restoreErr := operation.WithVolumeServerClient(false, source, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
if wasWritable { if wasWritable {
@@ -44,6 +44,8 @@ type tierDownloadCluster struct {
placement uint32 placement uint32
// staleTwin lists a remote-backed twin of the volume on another disk // staleTwin lists a remote-backed twin of the volume on another disk
staleTwin map[pb.ServerAddress]bool staleTwin map[pb.ServerAddress]bool
// markFails applies the readonly mark, then fails as a lost master notification does
markFails map[pb.ServerAddress]bool
} }
type tierDownloadTestServer struct { type tierDownloadTestServer struct {
@@ -104,6 +106,9 @@ func (s *tierDownloadTestServer) VolumeMarkReadonly(_ context.Context, req *volu
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
c.frozen[s.addr] = !req.CanDelete c.frozen[s.addr] = !req.CanDelete
if c.markFails[s.addr] {
return nil, fmt.Errorf("notify master: leader changed")
}
return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
} }
@@ -165,6 +170,7 @@ func newTierDownloadCluster(t *testing.T, states ...string) *tierDownloadCluster
writable: make(map[pb.ServerAddress]bool), writable: make(map[pb.ServerAddress]bool),
remounts: make(map[pb.ServerAddress]int), remounts: make(map[pb.ServerAddress]int),
staleTwin: make(map[pb.ServerAddress]bool), staleTwin: make(map[pb.ServerAddress]bool),
markFails: make(map[pb.ServerAddress]bool),
} }
for _, state := range states { for _, state := range states {
lis, err := net.Listen("tcp", "127.0.0.1:0") lis, err := net.Listen("tcp", "127.0.0.1:0")
@@ -385,3 +391,24 @@ func TestTierDownloadKeepsRemoteObjectForStaleTwinOnSource(t *testing.T) {
t.Errorf("remote object deleted while a twin on %s still references it", c.servers[0]) t.Errorf("remote object deleted while a twin on %s still references it", c.servers[0])
} }
} }
func TestTierDownloadRestoresSourceWhenMarkFailsAfterApplying(t *testing.T) {
for _, writable := range []bool{false, true} {
c := newTierDownloadCluster(t, tierReplicaLocal, tierReplicaRemote, tierReplicaRemote)
source := c.servers[0]
c.writable[source] = writable
c.markFails[source] = true
if err := c.run(t); err == nil {
t.Fatalf("writable=%v: tier download should fail when the readonly mark fails", writable)
}
if c.frozen[source] {
t.Errorf("writable=%v: source still blocks deletes after the failed mark", writable)
}
if c.writable[source] != writable {
t.Errorf("writable=%v: source writable state changed to %v", writable, c.writable[source])
}
if !c.objectExists {
t.Errorf("writable=%v: remote object deleted although no replica was replaced", writable)
}
}
}