volume: avoid read-only replica write targets (#11195)

* master: carry replica read-only state in volume lookups

* volume: refresh writable replica targets

* volume: preserve read-only replicas for deletes

* master: propagate read-only delete capability

* volume: target delete-capable replicas

* volume: honor configured HTTPS for replica deletes

* volume: reject insecure delete authorization forwarding

* master: broadcast delete capability changes

* volume: align Rust replica routing

* http: protect credentialed replica redirects

* master: preserve digest compatibility for delete capability

* volume: propagate read-only state in short heartbeats

* volume: report changed short volume state

* http: guard TLS client redirects

* master: announce mounted volume read-only state

* volume: replace changed identity deltas

* master: replace incremental volume layouts in order

* master: keep moved volume lookup available

* volume: announce read-only mounts

weed/server/master_grpc_server.go

@@ -88,11 +88,17 @@ func (ms *MasterServer) UnRegisterUuids(ip string, port int) {
 // goes on both lists: RemoteVids carries the classification, and NewVids keeps
 // a client too old to read RemoteVids from losing the volume altogether during
 // a rolling upgrade.
-func announceVolume(message *master_pb.VolumeLocation, vid uint32, isRemote bool) {
+func announceVolume(message *master_pb.VolumeLocation, vid uint32, isRemote, isReadOnly, readOnlyCanDelete bool) {
 	message.NewVids = append(message.NewVids, vid)
 	if isRemote {
 		message.RemoteVids = append(message.RemoteVids, vid)
 	}
+	if isReadOnly {
+		message.ReadOnlyVids = append(message.ReadOnlyVids, vid)
+		if readOnlyCanDelete {
+			message.ReadOnlyCanDeleteVids = append(message.ReadOnlyCanDeleteVids, vid)
+		}
+	}
 }
 
 func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
@@ -238,7 +244,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			for _, volInfo := range heartbeat.NewVolumes {
 				// The short form carries no remote-storage name, so the volume
 				// reads as local until a changed or full report names its tier.
-				announceVolume(message, volInfo.Id, false)
+				announceVolume(message, volInfo.Id, false, volInfo.ReadOnly, volInfo.ReadOnlyCanDelete)
 			}
 			for _, volInfo := range heartbeat.DeletedVolumes {
 				if !shouldBroadcastVolumeRemoval(dn, needle.VolumeId(volInfo.Id)) {
@@ -254,7 +260,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 				// Changed volumes include both newly-added replicas and existing
 				// replicas whose tier classification flipped, which the client
 				// has to be told about to refresh its replica priority.
-				announceVolume(message, uint32(v.Id), v.IsRemote())
+				announceVolume(message, uint32(v.Id), v.IsRemote(), v.ReadOnly, v.ReadOnlyCanDelete)
 			}
 		}
 
@@ -270,7 +276,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 
 			for _, v := range newVolumes {
 				glog.V(1).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
-				announceVolume(message, uint32(v.Id), v.IsRemote())
+				announceVolume(message, uint32(v.Id), v.IsRemote(), v.ReadOnly, v.ReadOnlyCanDelete)
 			}
 			// A full reconciliation is the digest mismatch recovery path, and
 			// the only way a re-tiered replica reaches the master without a
@@ -278,7 +284,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			// too is what stops the client keeping the old classification.
 			for _, v := range changedVolumes {
 				glog.V(1).Infof("master see tier/readonly change on volume %d from %s", uint32(v.Id), dn.Url())
-				announceVolume(message, uint32(v.Id), v.IsRemote())
+				announceVolume(message, uint32(v.Id), v.IsRemote(), v.ReadOnly, v.ReadOnlyCanDelete)
 			}
 			for _, v := range deletedVolumes {
 				glog.V(1).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())

weed/server/master_grpc_server_changed_volumes_tier_test.go

@@ -21,12 +21,12 @@ func changedTierVolume(id uint32, size uint64, remoteStorageName string) *master
 // ChangedVolumes heartbeat through announceVolume, the routing the server
 // itself uses, so the tests below assert what would really be broadcast
 // without standing up a gRPC stream.
-func announceChangedVolumes(topo *topology.Topology, dn *topology.DataNode, changed []*master_pb.VolumeInformationMessage) (newVids, remoteVids []uint32) {
+func announceChangedVolumes(topo *topology.Topology, dn *topology.DataNode, changed []*master_pb.VolumeInformationMessage) (newVids, remoteVids, readOnlyVids []uint32) {
 	message := &master_pb.VolumeLocation{}
 	for _, v := range topo.ApplyVolumeChanges(changed, dn) {
-		announceVolume(message, uint32(v.Id), v.IsRemote())
+		announceVolume(message, uint32(v.Id), v.IsRemote(), v.ReadOnly, v.ReadOnlyCanDelete)
 	}
-	return message.NewVids, message.RemoteVids
+	return message.NewVids, message.RemoteVids, message.ReadOnlyVids
 }
 
 // A replica that the heartbeat says has just been tiered to remote storage is
@@ -39,15 +39,18 @@ func TestChangedVolumesAnnounceLocalToRemoteTierTransition(t *testing.T) {
 		changedTestVolume(1, 1024),
 	}, dn)
 
-	newVids, remoteVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
-		changedTierVolume(1, 1024, "s3-bucket"),
-	})
+	v := changedTierVolume(1, 1024, "s3-bucket")
+	v.ReadOnly = true
+	newVids, remoteVids, readOnlyVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{v})
 	if !containsUint32(remoteVids, 1) {
 		t.Errorf("a local-to-remote transition was not announced as RemoteVids: %v", remoteVids)
 	}
 	if !containsUint32(newVids, 1) {
 		t.Errorf("a remote volume must stay on NewVids for clients that cannot read RemoteVids: %v", newVids)
 	}
+	if !containsUint32(readOnlyVids, 1) {
+		t.Errorf("a read-only transition was not announced as ReadOnlyVids: %v", readOnlyVids)
+	}
 }
 
 // A replica restored from remote storage back to a local disk flips the other
@@ -60,7 +63,7 @@ func TestChangedVolumesAnnounceRemoteToLocalTierTransition(t *testing.T) {
 		changedTierVolume(1, 1024, "s3-bucket"),
 	}, dn)
 
-	newVids, remoteVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
+	newVids, remoteVids, readOnlyVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
 		changedTestVolume(1, 1024),
 	})
 	if !containsUint32(newVids, 1) {
@@ -69,6 +72,9 @@ func TestChangedVolumesAnnounceRemoteToLocalTierTransition(t *testing.T) {
 	if containsUint32(remoteVids, 1) {
 		t.Errorf("a remote-to-local transition was routed as RemoteVids: %v", remoteVids)
 	}
+	if containsUint32(readOnlyVids, 1) {
+		t.Errorf("a writable transition was routed as ReadOnlyVids: %v", readOnlyVids)
+	}
 }
 
 // A heartbeat that re-reports an existing replica with the same tier
@@ -80,14 +86,43 @@ func TestChangedVolumesSuppressNoOpTierReport(t *testing.T) {
 		changedTestVolume(1, 1024),
 	}, dn)
 
-	newVids, remoteVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
+	newVids, remoteVids, readOnlyVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
 		changedTestVolume(1, 4096),
 	})
-	if containsUint32(newVids, 1) || containsUint32(remoteVids, 1) {
+	if containsUint32(newVids, 1) || containsUint32(remoteVids, 1) || containsUint32(readOnlyVids, 1) {
 		t.Errorf("a no-op tier report was broadcast: newVids=%v remoteVids=%v", newVids, remoteVids)
 	}
 }
 
+func TestChangedVolumesAnnounceSameTierReadOnlyTransition(t *testing.T) {
+	topo, dn := changedTestCluster(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{changedTestVolume(1, 1024)}, dn)
+
+	readOnly := changedTestVolume(1, 1024)
+	readOnly.ReadOnly = true
+	_, _, readOnlyVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{readOnly})
+	if !containsUint32(readOnlyVids, 1) {
+		t.Fatalf("writable-to-read-only transition was not announced: %v", readOnlyVids)
+	}
+	_, _, readOnlyVids = announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{changedTestVolume(1, 1024)})
+	if containsUint32(readOnlyVids, 1) {
+		t.Fatalf("read-only-to-writable transition retained ReadOnlyVids: %v", readOnlyVids)
+	}
+}
+
+func TestChangedVolumesAnnounceReadOnlyDeleteCapabilityTransition(t *testing.T) {
+	topo, dn := changedTestCluster(t)
+	initial := changedTestVolume(1, 1024)
+	initial.ReadOnly, initial.ReadOnlyCanDelete = true, true
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{initial}, dn)
+	next := changedTestVolume(1, 1024)
+	next.ReadOnly = true
+	_, _, readOnlyVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{next})
+	if !containsUint32(readOnlyVids, 1) {
+		t.Fatalf("read-only delete-capability transition was not announced: %v", readOnlyVids)
+	}
+}
+
 // A heartbeat that mixes a pure growth with a tier transition only announces
 // the tier-transitioned replica: a growth is local state, not a re-route, and
 // must not push other topology updates out of a bounded client queue.
@@ -98,7 +133,7 @@ func TestChangedVolumesAnnounceOnlyTierTransitions(t *testing.T) {
 		changedTestVolume(2, 1024),
 	}, dn)
 
-	newVids, remoteVids := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
+	newVids, remoteVids, _ := announceChangedVolumes(topo, dn, []*master_pb.VolumeInformationMessage{
 		changedTestVolume(1, 4096),              // pure growth
 		changedTierVolume(2, 1024, "s3-bucket"), // tier transition
 	})
@@ -133,25 +168,57 @@ func TestFullReconciliationAnnouncesTierTransition(t *testing.T) {
 		changedTestVolume(1, 1024),
 	}, dn)
 
-	newVids, remoteVids := announceFullReconciliation(topo, dn, []*master_pb.VolumeInformationMessage{
-		changedTierVolume(1, 1024, "s3-bucket"),
-	})
+	v := changedTierVolume(1, 1024, "s3-bucket")
+	v.ReadOnly = true
+	newVids, remoteVids, readOnlyVids := announceFullReconciliation(topo, dn, []*master_pb.VolumeInformationMessage{v})
 	if !containsUint32(remoteVids, 1) {
 		t.Errorf("a tier transition reported in a full reconciliation was not announced: newVids=%v remoteVids=%v", newVids, remoteVids)
 	}
 	if !containsUint32(newVids, 1) {
 		t.Errorf("a remote volume must stay on NewVids for clients that cannot read RemoteVids: %v", newVids)
 	}
+	if !containsUint32(readOnlyVids, 1) {
+		t.Errorf("a read-only transition was not announced as ReadOnlyVids: %v", readOnlyVids)
+	}
+}
+
+func TestFullReconciliationAnnounceSameTierReadOnlyTransition(t *testing.T) {
+	topo, dn := changedTestCluster(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{changedTestVolume(1, 1024)}, dn)
+
+	readOnly := changedTestVolume(1, 1024)
+	readOnly.ReadOnly = true
+	_, _, readOnlyVids := announceFullReconciliation(topo, dn, []*master_pb.VolumeInformationMessage{readOnly})
+	if !containsUint32(readOnlyVids, 1) {
+		t.Fatalf("writable-to-read-only reconciliation was not announced: %v", readOnlyVids)
+	}
+	_, _, readOnlyVids = announceFullReconciliation(topo, dn, []*master_pb.VolumeInformationMessage{changedTestVolume(1, 1024)})
+	if containsUint32(readOnlyVids, 1) {
+		t.Fatalf("read-only-to-writable reconciliation retained ReadOnlyVids: %v", readOnlyVids)
+	}
+}
+
+func TestFullReconciliationAnnounceReadOnlyDeleteCapabilityTransition(t *testing.T) {
+	topo, dn := changedTestCluster(t)
+	initial := changedTestVolume(1, 1024)
+	initial.ReadOnly, initial.ReadOnlyCanDelete = true, true
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{initial}, dn)
+	next := changedTestVolume(1, 1024)
+	next.ReadOnly = true
+	_, _, readOnlyVids := announceFullReconciliation(topo, dn, []*master_pb.VolumeInformationMessage{next})
+	if !containsUint32(readOnlyVids, 1) {
+		t.Fatalf("read-only delete-capability reconciliation was not announced: %v", readOnlyVids)
+	}
 }
 
 // announceFullReconciliation runs the same routing loop master_grpc_server's
 // SendHeartbeat does on a full Volumes heartbeat, including the changed-set
 // re-route added so digest-mismatch recovery propagates tier transitions.
-func announceFullReconciliation(topo *topology.Topology, dn *topology.DataNode, volumes []*master_pb.VolumeInformationMessage) (newVids, remoteVids []uint32) {
+func announceFullReconciliation(topo *topology.Topology, dn *topology.DataNode, volumes []*master_pb.VolumeInformationMessage) (newVids, remoteVids, readOnlyVids []uint32) {
 	message := &master_pb.VolumeLocation{}
 	newOnes, _, changedOnes := topo.SyncDataNodeRegistration(volumes, dn)
 	for _, v := range append(newOnes, changedOnes...) {
-		announceVolume(message, uint32(v.Id), v.IsRemote())
+		announceVolume(message, uint32(v.Id), v.IsRemote(), v.ReadOnly, v.ReadOnlyCanDelete)
 	}
-	return message.NewVids, message.RemoteVids
+	return message.NewVids, message.RemoteVids, message.ReadOnlyVids
 }

weed/server/master_grpc_server_volume.go

@@ -172,11 +172,13 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV
 			var locations []*master_pb.Location
 			for _, loc := range result.Locations {
 				locations = append(locations, &master_pb.Location{
-					Url:          loc.Url,
-					PublicUrl:    loc.PublicUrl,
-					DataCenter:   loc.DataCenter,
-					GrpcPort:     uint32(loc.GrpcPort),
-					DataInRemote: loc.DataInRemote,
+					Url:               loc.Url,
+					PublicUrl:         loc.PublicUrl,
+					DataCenter:        loc.DataCenter,
+					GrpcPort:          uint32(loc.GrpcPort),
+					DataInRemote:      loc.DataInRemote,
+					ReadOnly:          loc.ReadOnly,
+					ReadOnlyCanDelete: loc.ReadOnlyCanDelete,
 				})
 			}
 			var auth string

weed/server/master_server_handlers.go

@@ -92,11 +92,13 @@ func (ms *MasterServer) findVolumeLocation(collection, vid string) operation.Loo
 		machines, getVidLocationsErr := ms.MasterClient.GetVidLocations(vid)
 		for _, loc := range machines {
 			locations = append(locations, operation.Location{
-				Url:          loc.Url,
-				PublicUrl:    loc.PublicUrl,
-				DataCenter:   loc.DataCenter,
-				GrpcPort:     loc.GrpcPort,
-				DataInRemote: loc.DataInRemote,
+				Url:               loc.Url,
+				PublicUrl:         loc.PublicUrl,
+				DataCenter:        loc.DataCenter,
+				GrpcPort:          loc.GrpcPort,
+				DataInRemote:      loc.DataInRemote,
+				ReadOnly:          loc.ReadOnly,
+				ReadOnlyCanDelete: loc.ReadOnlyCanDelete,
 			})
 		}
 		err = getVidLocationsErr
@@ -121,16 +123,20 @@ func (ms *MasterServer) findVolumeLocation(collection, vid string) operation.Loo
 // EC volume holds shards rather than a volume record, so an absent record means
 // the read is local, never that the node should be left out of the answer.
 func topologyLocation(dn *topology.DataNode, vid needle.VolumeId) operation.Location {
-	dataInRemote := false
+	dataInRemote, readOnly, readOnlyCanDelete := false, false, false
 	if volInfo, lookupErr := dn.GetVolumesById(vid); lookupErr == nil {
 		dataInRemote = volInfo.IsRemote()
+		readOnly = volInfo.ReadOnly
+		readOnlyCanDelete = volInfo.ReadOnlyCanDelete
 	}
 	return operation.Location{
-		Url:          dn.Url(),
-		PublicUrl:    dn.PublicUrl,
-		DataCenter:   dn.GetDataCenterId(),
-		GrpcPort:     dn.GrpcPort,
-		DataInRemote: dataInRemote,
+		Url:               dn.Url(),
+		PublicUrl:         dn.PublicUrl,
+		DataCenter:        dn.GetDataCenterId(),
+		GrpcPort:          dn.GrpcPort,
+		DataInRemote:      dataInRemote,
+		ReadOnly:          readOnly,
+		ReadOnlyCanDelete: readOnlyCanDelete,
 	}
 }
 

weed/server/master_server_handlers_lookup_test.go

@@ -46,3 +46,14 @@ func TestVolumeLocationCarriesTheRemoteTier(t *testing.T) {
 		t.Errorf("a tiered volume was reported as local: %+v", loc)
 	}
 }
+
+func TestVolumeLocationCarriesReadOnly(t *testing.T) {
+	topo, dn := changedTestCluster(t)
+	v := changedTestVolume(1, 1024)
+	v.ReadOnly = true
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, dn)
+
+	if !topologyLocation(dn, needle.VolumeId(1)).ReadOnly {
+		t.Fatal("a read-only volume was reported as writable")
+	}
+}