mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de0c00b30e | ||
|
|
f5bfcd8814 | ||
|
|
dbe850f13d | ||
|
|
e0e24d6f49 | ||
|
|
c2b7e50b2b | ||
|
|
165d967dc8 | ||
|
|
16f4ea5822 | ||
|
|
d4a1685cf5 | ||
|
|
9f1febb2ac | ||
|
|
d78fccf453 | ||
|
|
945b4a6210 | ||
|
|
11d44177a0 | ||
|
|
be4bead284 | ||
|
|
f32be4c16e | ||
|
|
b01ff0b450 | ||
|
|
0160bc9cbc | ||
|
|
35ef8b3512 | ||
|
|
ca718f4181 | ||
|
|
cea59ad729 | ||
|
|
3ea9125e02 | ||
|
|
1f12bf6588 |
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
@@ -15,7 +16,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/telemetry"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
@@ -28,6 +31,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
||||
"github.com/seaweedfs/seaweedfs/weed/shell"
|
||||
@@ -71,7 +75,8 @@ type MasterServer struct {
|
||||
option *MasterOption
|
||||
guard *security.Guard
|
||||
|
||||
preallocateSize int64
|
||||
preallocateSize int64
|
||||
volumeIdAllocatorCloser io.Closer
|
||||
|
||||
Topo *topology.Topology
|
||||
vg *topology.VolumeGrowth
|
||||
@@ -124,6 +129,10 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
|
||||
option.VolumeGrowthDisabled = v.GetBool("master.volume_growth.disable")
|
||||
v.SetDefault("master.volume_growth.freeze_on_topology_mismatch", true)
|
||||
v.SetDefault("master.volume_growth.state_loss_recovery_wait", "0s")
|
||||
v.SetDefault("master.volume_id_allocator", "")
|
||||
v.SetDefault("master.volume_id_etcd.servers", "localhost:2379")
|
||||
v.SetDefault("master.volume_id_etcd.key_prefix", "/seaweedfs")
|
||||
v.SetDefault("master.volume_id_etcd.timeout", "3s")
|
||||
|
||||
topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1")
|
||||
topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2")
|
||||
@@ -162,6 +171,34 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
|
||||
ms.Topo.SetGrowthFreezeConfig(
|
||||
v.GetBool("master.volume_growth.freeze_on_topology_mismatch"),
|
||||
v.GetDuration("master.volume_growth.state_loss_recovery_wait"))
|
||||
ms.Topo.QuarantineVolumeFn = func(dn *topology.DataNode, vid needle.VolumeId, readOnly bool) error {
|
||||
err := operation.WithVolumeServerClient(false, dn.ServerAddress(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if readOnly {
|
||||
_, err := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: uint32(vid), CanDelete: true})
|
||||
return err
|
||||
}
|
||||
_, err := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: uint32(vid)})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("quarantine volume %d on %s readOnly=%v: %v", vid, dn.Id(), readOnly, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
switch v.GetString("master.volume_id_allocator") {
|
||||
case "":
|
||||
case "etcd":
|
||||
allocator, closer, err := newVolumeIdAllocator(v, ms.Topo)
|
||||
if err != nil {
|
||||
glog.Fatalf("volume id allocator: %v", err)
|
||||
}
|
||||
ms.volumeIdAllocatorCloser = closer
|
||||
ms.Topo.SetVolumeIdAllocator(allocator)
|
||||
default:
|
||||
glog.Fatalf("unknown master.volume_id_allocator %q", v.GetString("master.volume_id_allocator"))
|
||||
}
|
||||
ms.vg = topology.NewDefaultVolumeGrowth()
|
||||
glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")
|
||||
|
||||
@@ -653,6 +690,9 @@ func (ms *MasterServer) Shutdown() {
|
||||
if raftServer != nil {
|
||||
raftServer.Stop()
|
||||
}
|
||||
if ms.volumeIdAllocatorCloser != nil {
|
||||
ms.volumeIdAllocatorCloser.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *MasterServer) Reload() {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/client/pkg/v3/transport"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// newVolumeIdAllocator returns the allocator and an io.Closer to release the
|
||||
// etcd client on shutdown; the closer is nil for built-in allocation.
|
||||
func newVolumeIdAllocator(v *util.ViperProxy, topo *topology.Topology) (topology.VolumeIdAllocator, io.Closer, error) {
|
||||
servers := v.GetString("master.volume_id_etcd.servers")
|
||||
timeout, err := time.ParseDuration(v.GetString("master.volume_id_etcd.timeout"))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse master.volume_id_etcd.timeout: %w", err)
|
||||
}
|
||||
|
||||
var tlsConfig *tls.Config
|
||||
caFile := v.GetString("master.volume_id_etcd.tls_ca_file")
|
||||
certFile := v.GetString("master.volume_id_etcd.tls_client_crt_file")
|
||||
keyFile := v.GetString("master.volume_id_etcd.tls_client_key_file")
|
||||
if caFile != "" || certFile != "" || keyFile != "" {
|
||||
tlsInfo := transport.TLSInfo{
|
||||
CertFile: certFile,
|
||||
KeyFile: keyFile,
|
||||
TrustedCAFile: caFile,
|
||||
}
|
||||
if tlsConfig, err = tlsInfo.ClientConfig(); err != nil {
|
||||
return nil, nil, fmt.Errorf("etcd TLS client configuration error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
client, err := clientv3.New(clientv3.Config{
|
||||
Endpoints: strings.Split(servers, ","),
|
||||
Username: v.GetString("master.volume_id_etcd.username"),
|
||||
Password: v.GetString("master.volume_id_etcd.password"),
|
||||
DialTimeout: timeout,
|
||||
TLS: tlsConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("connect volume id etcd: %w", err)
|
||||
}
|
||||
|
||||
glog.Infof("volume id allocator: etcd %s", servers)
|
||||
allocator := topology.NewEtcdVolumeIdAllocator(
|
||||
client,
|
||||
v.GetString("master.volume_id_etcd.key_prefix"),
|
||||
timeout,
|
||||
topo.GetMaxVolumeId)
|
||||
return allocator, allocator, nil
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandVolumeCheckDuplicateIds{})
|
||||
}
|
||||
|
||||
type commandVolumeCheckDuplicateIds struct {
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDuplicateIds) Name() string {
|
||||
return "volume.check.duplicate_ids"
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDuplicateIds) Help() string {
|
||||
return `find volume ids registered under more than one layout
|
||||
|
||||
volume.check.duplicate_ids
|
||||
|
||||
Volume ids are global. The same id under two collections, replica
|
||||
placements, TTLs, or disk types means the allocator reused an id, for
|
||||
example after master state loss. See volume.growth.resume.
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDuplicateIds) HasTag(CommandTag) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *commandVolumeCheckDuplicateIds) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// vid -> layout key -> locations; ec shards tracked per collection since
|
||||
// shards plus regular copies of the same encoded volume are legitimate
|
||||
seen := make(map[needle.VolumeId]map[string][]string)
|
||||
ecSeen := make(map[needle.VolumeId]map[string][]string)
|
||||
collections := make(map[needle.VolumeId]map[string]bool)
|
||||
ecCollections := make(map[needle.VolumeId]map[string]bool)
|
||||
add := func(m map[needle.VolumeId]map[string][]string, id uint32, key, location string) {
|
||||
layouts := m[needle.VolumeId(id)]
|
||||
if layouts == nil {
|
||||
layouts = make(map[string][]string)
|
||||
m[needle.VolumeId(id)] = layouts
|
||||
}
|
||||
layouts[key] = append(layouts[key], location)
|
||||
}
|
||||
addCollection := func(m map[needle.VolumeId]map[string]bool, id uint32, collection string) {
|
||||
if m[needle.VolumeId(id)] == nil {
|
||||
m[needle.VolumeId(id)] = make(map[string]bool)
|
||||
}
|
||||
m[needle.VolumeId(id)][collection] = true
|
||||
}
|
||||
for _, dc := range topoInfo.DataCenterInfos {
|
||||
for _, r := range dc.RackInfos {
|
||||
for _, dn := range r.DataNodeInfos {
|
||||
for diskKey, di := range dn.DiskInfos {
|
||||
for _, vi := range di.VolumeInfos {
|
||||
rp, _ := super_block.NewReplicaPlacementFromByte(byte(vi.ReplicaPlacement))
|
||||
add(seen, vi.Id, fmt.Sprintf("collection=%q replication=%s ttl=%s disk=%s",
|
||||
vi.Collection, rp.String(), needle.LoadTTLFromUint32(vi.Ttl), vi.DiskType),
|
||||
fmt.Sprintf("%s %s", dn.Id, diskKey))
|
||||
addCollection(collections, vi.Id, vi.Collection)
|
||||
}
|
||||
for _, si := range di.EcShardInfos {
|
||||
add(ecSeen, si.Id, fmt.Sprintf("collection=%q ec_shards", si.Collection),
|
||||
fmt.Sprintf("%s %s disk=%s", dn.Id, diskKey, si.DiskType))
|
||||
addCollection(ecCollections, si.Id, si.Collection)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vids := make(map[needle.VolumeId]struct{}, len(seen)+len(ecSeen))
|
||||
for vid := range seen {
|
||||
vids[vid] = struct{}{}
|
||||
}
|
||||
for vid := range ecSeen {
|
||||
vids[vid] = struct{}{}
|
||||
}
|
||||
|
||||
duplicates := 0
|
||||
for _, vid := range sortedKeys(vids) {
|
||||
layouts := seen[vid]
|
||||
shardLayouts := ecSeen[vid]
|
||||
ecForeign := false
|
||||
if len(collections[vid]) > 0 {
|
||||
for collection := range ecCollections[vid] {
|
||||
if !collections[vid][collection] {
|
||||
ecForeign = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(layouts) < 2 && len(shardLayouts) < 2 && !ecForeign {
|
||||
continue
|
||||
}
|
||||
duplicates++
|
||||
fmt.Fprintf(writer, "volume %d is registered under %d layouts:\n", vid, len(layouts)+len(shardLayouts))
|
||||
for _, key := range sortedKeysString(layouts) {
|
||||
fmt.Fprintf(writer, " %-60s %v\n", key, layouts[key])
|
||||
}
|
||||
for _, key := range sortedKeysString(shardLayouts) {
|
||||
fmt.Fprintf(writer, " %-60s %v\n", key, shardLayouts[key])
|
||||
}
|
||||
}
|
||||
if duplicates == 0 {
|
||||
fmt.Fprintf(writer, "no duplicate volume ids found\n")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[needle.VolumeId]V) []needle.VolumeId {
|
||||
keys := make([]needle.VolumeId, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func sortedKeysString[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -109,6 +109,14 @@ var (
|
||||
Help: "1 while volume growth is frozen after detected master state loss",
|
||||
})
|
||||
|
||||
MasterDuplicateVolumeIds = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: subsystemMaster,
|
||||
Name: "duplicate_volume_ids",
|
||||
Help: "volume id registered under two different layouts",
|
||||
}, []string{"collection", "volumeId"})
|
||||
|
||||
MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
@@ -1000,6 +1008,7 @@ func init() {
|
||||
Gather.MustRegister(MasterRaftIsleader)
|
||||
Gather.MustRegister(MasterAdminLock)
|
||||
Gather.MustRegister(MasterVolumeGrowthFrozen)
|
||||
Gather.MustRegister(MasterDuplicateVolumeIds)
|
||||
Gather.MustRegister(MasterReceivedHeartbeatCounter)
|
||||
Gather.MustRegister(MasterLeaderChangeCounter)
|
||||
Gather.MustRegister(MasterReplicaPlacementMismatch)
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func isWritable(t *testing.T, topo *Topology, v storage.VolumeInfo) bool {
|
||||
t.Helper()
|
||||
vl := topo.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType))
|
||||
for _, id := range vl.CloneWritableVolumes() {
|
||||
if id == v.Id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestDuplicateVolumeIdAcrossCollections(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn2 := NewDataNode("server2")
|
||||
|
||||
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
||||
topo.RegisterVolumeLayout(v1, dn1)
|
||||
topo.RegisterVolumeLayout(v2, dn2)
|
||||
topo.flagDuplicateVolumeId(v2, dn2)
|
||||
|
||||
if isWritable(t, topo, v2) {
|
||||
t.Fatal("duplicate newcomer should be quarantined out of the writable list")
|
||||
}
|
||||
if isWritable(t, topo, v1) {
|
||||
t.Fatal("either copy could be the impostor; the earlier copy must be quarantined too")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameLayoutReplicaIsNotDuplicate(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("001")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn1.Ip = "10.0.0.1"
|
||||
dn1.Port = 8080
|
||||
dn2 := NewDataNode("server2")
|
||||
dn2.Ip = "10.0.0.2"
|
||||
dn2.Port = 8080
|
||||
|
||||
v := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
topo.RegisterVolumeLayout(v, dn1)
|
||||
topo.RegisterVolumeLayout(v, dn2)
|
||||
topo.flagDuplicateVolumeId(v, dn2)
|
||||
|
||||
if !isWritable(t, topo, v) {
|
||||
t.Fatal("a same-layout replica is a legitimate copy, not a duplicate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameNodeVolumeMoveIsNotDuplicate(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
dn := NewDataNode("server1")
|
||||
|
||||
old := storage.VolumeInfo{Id: 7, Collection: "c", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
moved := storage.VolumeInfo{Id: 7, Collection: "c", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: "ssd"}
|
||||
dn.UpdateVolumes([]storage.VolumeInfo{old})
|
||||
topo.RegisterVolumeLayout(old, dn)
|
||||
|
||||
// the move registers the new layout before the old entry is removed
|
||||
dn.UpdateVolumes([]storage.VolumeInfo{moved})
|
||||
topo.RegisterVolumeLayout(moved, dn)
|
||||
topo.flagDuplicateVolumeId(moved, dn)
|
||||
|
||||
if !isWritable(t, topo, moved) {
|
||||
t.Fatal("a same-node disk move is not a duplicate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuarantineReleasedWhenConflictResolves(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn2 := NewDataNode("server2")
|
||||
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
||||
topo.RegisterVolumeLayout(v1, dn1)
|
||||
topo.RegisterVolumeLayout(v2, dn2)
|
||||
topo.flagDuplicateVolumeId(v2, dn2)
|
||||
|
||||
topo.UnRegisterVolumeLayout(v2, dn2)
|
||||
|
||||
if !isWritable(t, topo, v1) {
|
||||
t.Fatal("the surviving copy must be released once the conflict is gone")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuarantineSurvivesResync(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn2 := NewDataNode("server2")
|
||||
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
||||
topo.RegisterVolumeLayout(v1, dn1)
|
||||
topo.RegisterVolumeLayout(v2, dn2)
|
||||
topo.flagDuplicateVolumeId(v2, dn2)
|
||||
|
||||
// a resync from the same server re-reports the volume as writable;
|
||||
// the quarantine must keep it out of the writable list
|
||||
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
|
||||
Id: 7, Collection: "b", ReplicaPlacement: 0, Version: 3,
|
||||
}}, dn2)
|
||||
|
||||
if isWritable(t, topo, v2) {
|
||||
t.Fatal("resync must not clear the quarantine")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicationChangeIsNotDuplicate(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rpOld, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
rpNew, _ := super_block.NewReplicaPlacementFromString("001")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn1.Ip = "10.0.0.1"
|
||||
dn1.Port = 8080
|
||||
dn2 := NewDataNode("server2")
|
||||
dn2.Ip = "10.0.0.2"
|
||||
dn2.Port = 8080
|
||||
|
||||
vOld := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rpOld, Ttl: needle.EMPTY_TTL}
|
||||
vNew := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rpNew, Ttl: needle.EMPTY_TTL}
|
||||
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{vOld})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{vNew})
|
||||
topo.RegisterVolumeLayout(vOld, dn1)
|
||||
topo.RegisterVolumeLayout(vNew, dn2)
|
||||
topo.flagDuplicateVolumeId(vNew, dn2)
|
||||
|
||||
if topo.isQuarantined(dn2, vNew.Id) || topo.isQuarantined(dn1, vOld.Id) {
|
||||
t.Fatal("an rp-only layout difference is a replication change in flight, not a duplicate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateVolumeIdAcrossTtls(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
ttl, _ := needle.ReadTTL("3d")
|
||||
|
||||
dn1 := NewDataNode("server1")
|
||||
dn1.Ip = "10.0.0.1"
|
||||
dn1.Port = 8080
|
||||
dn2 := NewDataNode("server2")
|
||||
dn2.Ip = "10.0.0.2"
|
||||
dn2.Port = 8080
|
||||
|
||||
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
||||
v2 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: ttl}
|
||||
|
||||
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
||||
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
||||
topo.RegisterVolumeLayout(v1, dn1)
|
||||
topo.RegisterVolumeLayout(v2, dn2)
|
||||
topo.flagDuplicateVolumeId(v2, dn2)
|
||||
|
||||
if !topo.isQuarantined(dn2, v2.Id) || !topo.isQuarantined(dn1, v1.Id) {
|
||||
t.Fatal("same collection with a different ttl is still a reused volume id")
|
||||
}
|
||||
}
|
||||
+267
-31
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -89,6 +90,7 @@ type Topology struct {
|
||||
lastPersistedMaxVolumeId atomic.Uint32
|
||||
lastPersistedTopologyId atomic.Value
|
||||
lastPersistedGrowthFrozen atomic.Bool
|
||||
volumeIdAllocator VolumeIdAllocator
|
||||
|
||||
growthFrozen atomic.Bool
|
||||
growthFrozenAt atomic.Int64
|
||||
@@ -99,6 +101,55 @@ type Topology struct {
|
||||
|
||||
reportedTopologyIdsLock sync.Mutex
|
||||
pendingReportedTopologyIds map[string]struct{}
|
||||
|
||||
// quarantinedVolumeIds are (node, vid) pairs force-read-only because the
|
||||
// vid collided across layouts. Kept separate from the reported flag so
|
||||
// later heartbeats from the same server cannot quietly undo it.
|
||||
quarantinedVolumeIds sync.Map
|
||||
|
||||
// QuarantineVolumeFn pushes the quarantine onto the volume server itself
|
||||
// (VolumeMarkReadonly/VolumeMarkWritable). nil keeps quarantine at the
|
||||
// layout level only.
|
||||
QuarantineVolumeFn func(dn *DataNode, vid needle.VolumeId, readOnly bool) error
|
||||
}
|
||||
|
||||
type quarantinedVolume struct {
|
||||
dnId NodeId
|
||||
vid needle.VolumeId
|
||||
}
|
||||
|
||||
type quarantinedVolumeState struct {
|
||||
reported bool
|
||||
enforced bool
|
||||
}
|
||||
|
||||
func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool {
|
||||
_, ok := t.quarantinedVolumeIds.Load(quarantinedVolume{dn.Id(), vid})
|
||||
return ok
|
||||
}
|
||||
|
||||
// enforceQuarantine keeps the latest reported read-only flag for a quarantined
|
||||
// copy so a later release restores the server's true state. The stored report
|
||||
// keeps the server's flag: quarantine lives at the layout level, and folding
|
||||
// it into the record here would make the master's volume digest permanently
|
||||
// disagree with the server's and force a full volume list on every heartbeat.
|
||||
func (t *Topology) enforceQuarantine(dn *DataNode, vi storage.VolumeInfo) storage.VolumeInfo {
|
||||
q := quarantinedVolume{dn.Id(), vi.Id}
|
||||
if v, ok := t.quarantinedVolumeIds.Load(q); ok {
|
||||
state := v.(quarantinedVolumeState)
|
||||
if !state.enforced {
|
||||
state.reported = vi.ReadOnly
|
||||
}
|
||||
if t.QuarantineVolumeFn != nil && (!state.enforced || !vi.ReadOnly) {
|
||||
// The mark never landed, or the server lost it (restart, manual
|
||||
// unquarantine); push it while the conflict stands.
|
||||
if t.QuarantineVolumeFn(dn, vi.Id, true) == nil {
|
||||
state.enforced = true
|
||||
}
|
||||
}
|
||||
t.quarantinedVolumeIds.Store(q, state)
|
||||
}
|
||||
return vi
|
||||
}
|
||||
|
||||
func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology {
|
||||
@@ -421,6 +472,22 @@ func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
|
||||
return 0, fmt.Errorf("volume growth is frozen after master state loss; verify the cluster inventory, then run: volume.growth.resume")
|
||||
}
|
||||
}
|
||||
if a := t.volumeIdAllocator; a != nil {
|
||||
vid, err := a.NextVolumeId()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := t.PersistMaxVolumeId(vid); err != nil {
|
||||
return 0, fmt.Errorf("can not assign next volume id %d: %w", vid, err)
|
||||
}
|
||||
// Still record through raft so followers' boundaries and mirrors track
|
||||
// allocations made against the external allocator.
|
||||
if err := t.raftApply(NewMaxVolumeIdCommand(vid, t.GetTopologyId())); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
t.UpAdjustMaxVolumeId(vid)
|
||||
return vid, nil
|
||||
}
|
||||
vid := t.GetMaxVolumeId()
|
||||
next := vid.Next()
|
||||
|
||||
@@ -428,25 +495,29 @@ func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
|
||||
return 0, fmt.Errorf("persist max volume id %d: %w", next, err)
|
||||
}
|
||||
|
||||
t.RaftServerAccessLock.RLock()
|
||||
defer t.RaftServerAccessLock.RUnlock()
|
||||
|
||||
if t.RaftServer != nil {
|
||||
if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next, t.GetTopologyId())); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else if t.HashicorpRaft != nil {
|
||||
b, err := json.Marshal(NewMaxVolumeIdCommand(next, t.GetTopologyId()))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed marshal NewMaxVolumeIdCommand: %+v", err)
|
||||
}
|
||||
if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
|
||||
return 0, future.Error()
|
||||
}
|
||||
if err := t.raftApply(NewMaxVolumeIdCommand(next, t.GetTopologyId())); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (t *Topology) raftApply(cmd *MaxVolumeIdCommand) error {
|
||||
t.RaftServerAccessLock.RLock()
|
||||
defer t.RaftServerAccessLock.RUnlock()
|
||||
if t.RaftServer != nil {
|
||||
_, err := t.RaftServer.Do(cmd)
|
||||
return err
|
||||
}
|
||||
if t.HashicorpRaft != nil {
|
||||
b, err := json.Marshal(cmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed marshal %s: %+v", cmd.CommandName(), err)
|
||||
}
|
||||
return t.HashicorpRaft.Apply(b, time.Second).Error()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultNeedleSizeEstimate is the fallback per-file-ID size estimate when
|
||||
// the client does not provide an expected data size.
|
||||
const DefaultNeedleSizeEstimate uint64 = 1024 * 1024 // 1 MB
|
||||
@@ -591,9 +662,15 @@ func (t *Topology) DeleteCollection(collectionName string) {
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
var vids []needle.VolumeId
|
||||
for _, vl := range collection.(*Collection).GetAllVolumeLayouts() {
|
||||
vids = append(vids, vl.ListVids()...)
|
||||
vl.releaseLookupOwnership()
|
||||
}
|
||||
// a vid colliding with a copy in this collection is no longer a duplicate
|
||||
for _, vid := range vids {
|
||||
t.releaseResolvedQuarantine(vid)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Topology) DeleteLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) {
|
||||
@@ -612,15 +689,171 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
|
||||
for {
|
||||
vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
||||
if vl.RegisterVolume(&v, dn) {
|
||||
if t.isQuarantined(dn, v.Id) {
|
||||
vl.SetVolumeQuarantined(dn, v.Id)
|
||||
}
|
||||
vl.EnsureCorrectWritables(&v)
|
||||
go t.flagDuplicateVolumeId(v, dn)
|
||||
return
|
||||
}
|
||||
// Dropped with its collection; the next lookup creates a fresh one.
|
||||
}
|
||||
}
|
||||
|
||||
// flagDuplicateVolumeId reports and quarantines v.Id when another data node
|
||||
// already serves it under a different layout key. It runs after registration
|
||||
// so a legitimate same-node move (disk type or collection change) settles
|
||||
// first and does not trip it. Every copy across the conflicting layouts is
|
||||
// quarantined: either could be the impostor, and arrival order must not
|
||||
// decide which copy stays writable.
|
||||
func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) {
|
||||
hits := t.findDuplicateVolumeLayouts(v, dn)
|
||||
if len(hits) == 0 {
|
||||
return
|
||||
}
|
||||
// A conflict could resolve between the scan above and the marks below;
|
||||
// re-run the release so a stale quarantine is cleared instead of kept.
|
||||
defer t.releaseResolvedQuarantine(v.Id)
|
||||
stats.MasterDuplicateVolumeIds.WithLabelValues(v.Collection, v.Id.String()).Inc()
|
||||
var otherKeys []string
|
||||
for _, h := range hits {
|
||||
otherKeys = append(otherKeys, h.layoutKey())
|
||||
}
|
||||
glog.Errorf("duplicate volume id %d: %s reports it under %s but other data nodes also serve it under %s; marking all copies read-only", v.Id, dn.Id(), volumeLayoutKey(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)), strings.Join(otherKeys, ", "))
|
||||
quarantine := func(n *DataNode, vl *VolumeLayout) {
|
||||
q := quarantinedVolume{n.Id(), v.Id}
|
||||
reported := false
|
||||
if vi, err := n.GetVolumesById(v.Id); err == nil {
|
||||
reported = vi.ReadOnly
|
||||
}
|
||||
t.quarantinedVolumeIds.LoadOrStore(q, quarantinedVolumeState{reported: reported})
|
||||
if vl != nil {
|
||||
vl.SetVolumeQuarantined(n, v.Id)
|
||||
}
|
||||
if t.QuarantineVolumeFn != nil && t.QuarantineVolumeFn(n, v.Id, true) == nil {
|
||||
if val, ok := t.quarantinedVolumeIds.Load(q); ok {
|
||||
state := val.(quarantinedVolumeState)
|
||||
state.enforced = true
|
||||
t.quarantinedVolumeIds.Store(q, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
quarantine(dn, t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)))
|
||||
for _, h := range hits {
|
||||
for _, n := range h.nodes {
|
||||
quarantine(n, h.vl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// releaseResolvedQuarantine lifts read-only marks once the conflicting copy
|
||||
// is gone, so a finished move does not leave the surviving copy quarantined.
|
||||
func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) {
|
||||
t.quarantinedVolumeIds.Range(func(key, val any) bool {
|
||||
q := key.(quarantinedVolume)
|
||||
if q.vid != vid {
|
||||
return true
|
||||
}
|
||||
dn, vl, vi := t.findQuarantinedVolume(q.dnId, vid)
|
||||
if dn == nil {
|
||||
t.quarantinedVolumeIds.Delete(key)
|
||||
return true
|
||||
}
|
||||
if hits := t.findDuplicateVolumeLayouts(vi, dn); len(hits) > 0 {
|
||||
return true
|
||||
}
|
||||
state, _ := val.(quarantinedVolumeState)
|
||||
if state.reported {
|
||||
dn.SetVolumeReadOnly(vid, true)
|
||||
vl.SetReplicaReadOnlyFlag(dn, vid, true)
|
||||
} else {
|
||||
vl.SetVolumeWritable(dn, vid)
|
||||
if state.enforced && t.QuarantineVolumeFn != nil {
|
||||
if err := t.QuarantineVolumeFn(dn, vid, false); err != nil {
|
||||
// The server still rejects writes; keep the mark so a
|
||||
// later release pass retries the writable push.
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
t.quarantinedVolumeIds.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Topology) findQuarantinedVolume(dnId NodeId, vid needle.VolumeId) (*DataNode, *VolumeLayout, storage.VolumeInfo) {
|
||||
for _, item := range t.collectionMap.Items() {
|
||||
for _, vl := range item.(*Collection).GetAllVolumeLayouts() {
|
||||
for _, n := range vl.Lookup(vid) {
|
||||
if n.Id() == dnId {
|
||||
vi, _ := n.GetVolumesById(vid)
|
||||
return n, vl, vi
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil, storage.VolumeInfo{}
|
||||
}
|
||||
|
||||
// duplicateHit is one layout that holds v.Id on data nodes other than the
|
||||
// reporter's.
|
||||
type duplicateHit struct {
|
||||
collection string
|
||||
vl *VolumeLayout
|
||||
nodes []*DataNode
|
||||
}
|
||||
|
||||
func (h duplicateHit) layoutKey() string {
|
||||
return volumeLayoutKey(h.collection, h.vl.rp, h.vl.ttl, h.vl.diskType)
|
||||
}
|
||||
|
||||
func volumeLayoutKey(collection string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) string {
|
||||
return fmt.Sprintf("collection=%q rp=%s ttl=%s disk=%s", collection, rp, ttl, diskType)
|
||||
}
|
||||
|
||||
func ttlEqual(a, b *needle.TTL) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
}
|
||||
return a.ToUint32() == b.ToUint32()
|
||||
}
|
||||
|
||||
// findDuplicateVolumeLayouts reports every layout that already holds v.Id on
|
||||
// data nodes other than dn. Volume ids are global, so the same id held by two
|
||||
// nodes under two layouts means the allocator reused an id.
|
||||
func (t *Topology) findDuplicateVolumeLayouts(v storage.VolumeInfo, dn *DataNode) []duplicateHit {
|
||||
if v.ReplicaPlacement == nil {
|
||||
return nil
|
||||
}
|
||||
diskType := types.ToDiskType(v.DiskType)
|
||||
var hits []duplicateHit
|
||||
for _, item := range t.collectionMap.Items() {
|
||||
c := item.(*Collection)
|
||||
for _, vl := range c.GetAllVolumeLayouts() {
|
||||
if c.Name == v.Collection && vl.diskType == diskType && ttlEqual(vl.ttl, v.Ttl) {
|
||||
// The reporter's own layout, or an rp-only difference:
|
||||
// a replication change rolls through the nodes one at a
|
||||
// time, so the same volume legitimately reports the old
|
||||
// and new rp while it settles.
|
||||
continue
|
||||
}
|
||||
var others []*DataNode
|
||||
for _, n := range vl.Lookup(v.Id) {
|
||||
if n != dn {
|
||||
others = append(others, n)
|
||||
}
|
||||
}
|
||||
if len(others) > 0 {
|
||||
hits = append(hits, duplicateHit{collection: c.Name, vl: vl, nodes: others})
|
||||
}
|
||||
}
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
|
||||
glog.Infof("removing volume info: %+v from %v", v, dn.id)
|
||||
defer t.releaseResolvedQuarantine(v.Id)
|
||||
diskType := types.ToDiskType(v.DiskType)
|
||||
volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
||||
volumeLayout.UnRegisterVolume(&v, dn)
|
||||
@@ -745,6 +978,9 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati
|
||||
glog.V(0).Infof("Fail to convert joined volume information: %v", err)
|
||||
}
|
||||
}
|
||||
for i := range volumeInfos {
|
||||
volumeInfos[i] = t.enforceQuarantine(dn, volumeInfos[i])
|
||||
}
|
||||
// find out the delta volumes
|
||||
newVolumes, deletedVolumes, changedVolumes = dn.UpdateVolumes(volumeInfos)
|
||||
for _, v := range newVolumes {
|
||||
@@ -757,7 +993,7 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati
|
||||
if v.ReplicaPlacement == nil {
|
||||
continue
|
||||
}
|
||||
t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)).SetReplicaReadOnlyFlag(dn, v.Id, v.ReadOnly)
|
||||
t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)).SetReplicaReadOnlyFlag(dn, v.Id, v.ReadOnly || t.isQuarantined(dn, v.Id))
|
||||
}
|
||||
// Update effective sizes for all reported volumes (decay pending estimates).
|
||||
// If decay brings a volume eagerly removed by RecordAssign back under the
|
||||
@@ -781,6 +1017,7 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati
|
||||
// one, which the calls below must use too.
|
||||
vl = t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
||||
}
|
||||
go t.flagDuplicateVolumeId(v, dn)
|
||||
// Volumes new to the disk map were registered above, so reaching
|
||||
// here means only the lookup index had lost it. Clients were told
|
||||
// it went when the node dropped out, so the repair has to tell them
|
||||
@@ -804,7 +1041,7 @@ func (t *Topology) IncrementalSyncDataNodeRegistration(newVolumes, deletedVolume
|
||||
glog.V(0).Infof("NewVolumeInfoFromShort %v: %v", v, err)
|
||||
continue
|
||||
}
|
||||
newVis = append(newVis, vi)
|
||||
newVis = append(newVis, t.enforceQuarantine(dn, vi))
|
||||
}
|
||||
for _, v := range deletedVolumes {
|
||||
vi, err := storage.NewVolumeInfoFromShort(v)
|
||||
@@ -865,7 +1102,7 @@ func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMess
|
||||
glog.V(0).Infof("Fail to convert changed volume information: %v", err)
|
||||
continue
|
||||
}
|
||||
volumeInfos = append(volumeInfos, vi)
|
||||
volumeInfos = append(volumeInfos, t.enforceQuarantine(dn, vi))
|
||||
}
|
||||
|
||||
for _, vi := range volumeInfos {
|
||||
@@ -885,11 +1122,14 @@ func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMess
|
||||
// Dropped with its collection; the next lookup creates a fresh one.
|
||||
vl = t.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, vi.Ttl, types.ToDiskType(vi.DiskType))
|
||||
}
|
||||
if becameServable {
|
||||
go t.flagDuplicateVolumeId(vi, dn)
|
||||
}
|
||||
if isNew || becameServable || tierTransition || isChanged {
|
||||
newVolumes = append(newVolumes, vi)
|
||||
}
|
||||
if isChanged {
|
||||
vl.SetReplicaReadOnlyFlag(dn, vi.Id, vi.ReadOnly)
|
||||
vl.SetReplicaReadOnlyFlag(dn, vi.Id, vi.ReadOnly || t.isQuarantined(dn, vi.Id))
|
||||
}
|
||||
vl.UpdateOversizedState(&vi, dn)
|
||||
if vl.UpdateVolumeSize(vi.Id, vi.Size, vi.CompactRevision, true) {
|
||||
@@ -982,6 +1222,10 @@ func (t *Topology) SetGrowthFreezeConfig(freezeOnMismatch bool, recoveryWait tim
|
||||
t.stateLossRecoveryWait.Store(int64(recoveryWait))
|
||||
}
|
||||
|
||||
func (t *Topology) SetVolumeIdAllocator(a VolumeIdAllocator) {
|
||||
t.volumeIdAllocator = a
|
||||
}
|
||||
|
||||
// CheckReportedTopologyId freezes growth when a volume server reports a
|
||||
// topology id different from this cluster's: the topology id survives raft
|
||||
// state cleanup, so a mismatch means the master's state was rebuilt (or a
|
||||
@@ -1062,21 +1306,13 @@ func (t *Topology) SetGrowthFrozen(frozen bool) {
|
||||
func (t *Topology) applyGrowthFreeze(frozen bool) {
|
||||
cmd := NewMaxVolumeIdCommand(t.GetMaxVolumeId(), t.GetTopologyId())
|
||||
cmd.GrowthFrozen = &frozen
|
||||
t.RaftServerAccessLock.RLock()
|
||||
defer t.RaftServerAccessLock.RUnlock()
|
||||
if t.RaftServer != nil {
|
||||
if _, err := t.RaftServer.Do(cmd); err != nil {
|
||||
if t.RaftServer != nil || t.HashicorpRaft != nil {
|
||||
if err := t.raftApply(cmd); err != nil {
|
||||
glog.Warningf("apply growth freeze %v: %v", frozen, err)
|
||||
}
|
||||
} else if t.HashicorpRaft != nil {
|
||||
if b, err := json.Marshal(cmd); err != nil {
|
||||
glog.Warningf("marshal growth freeze %v: %v", frozen, err)
|
||||
} else if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
|
||||
glog.Warningf("apply growth freeze %v: %v", frozen, future.Error())
|
||||
}
|
||||
} else {
|
||||
t.SetGrowthFrozen(frozen)
|
||||
return
|
||||
}
|
||||
t.SetGrowthFrozen(frozen)
|
||||
}
|
||||
|
||||
// SetTopologyInitPending gates allocation while a topology id is being
|
||||
|
||||
@@ -118,6 +118,8 @@ func (t *Topology) UnRegisterDataNode(dn *DataNode) {
|
||||
diskType := types.ToDiskType(v.DiskType)
|
||||
vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
||||
vl.SetVolumeUnavailable(dn, v.Id)
|
||||
// this node may have been the conflicting copy pinning a quarantine
|
||||
defer t.releaseResolvedQuarantine(v.Id)
|
||||
}
|
||||
|
||||
// unregister ec shards when volume server disconnected
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
// VolumeIdAllocator hands out the next volume id. The builtin path advances
|
||||
// the raft-applied high-water mark; an external implementation makes the
|
||||
// boundary survive total master state loss.
|
||||
type VolumeIdAllocator interface {
|
||||
NextVolumeId() (needle.VolumeId, error)
|
||||
}
|
||||
|
||||
// EtcdVolumeIdAllocator keeps the volume id counter in a single etcd key,
|
||||
// advanced by compare-and-swap. Unlike the raft/mirror/echo layers it has no
|
||||
// residual window: an id cannot be issued without the etcd write committing.
|
||||
type EtcdVolumeIdAllocator struct {
|
||||
kv clientv3.KV
|
||||
client *clientv3.Client
|
||||
key string
|
||||
timeout time.Duration
|
||||
seed func() needle.VolumeId
|
||||
}
|
||||
|
||||
func NewEtcdVolumeIdAllocator(client *clientv3.Client, keyPrefix string, timeout time.Duration, seed func() needle.VolumeId) *EtcdVolumeIdAllocator {
|
||||
a := newEtcdVolumeIdAllocator(client.KV, keyPrefix, timeout, seed)
|
||||
a.client = client
|
||||
return a
|
||||
}
|
||||
|
||||
func newEtcdVolumeIdAllocator(kv clientv3.KV, keyPrefix string, timeout time.Duration, seed func() needle.VolumeId) *EtcdVolumeIdAllocator {
|
||||
return &EtcdVolumeIdAllocator{
|
||||
kv: kv,
|
||||
key: strings.TrimRight(keyPrefix, "/") + "/max_volume_id",
|
||||
timeout: timeout,
|
||||
seed: seed,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *EtcdVolumeIdAllocator) Close() error {
|
||||
if a.client == nil {
|
||||
return nil
|
||||
}
|
||||
return a.client.Close()
|
||||
}
|
||||
|
||||
func (a *EtcdVolumeIdAllocator) NextVolumeId() (needle.VolumeId, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), a.timeout)
|
||||
defer cancel()
|
||||
for {
|
||||
resp, err := a.kv.Get(ctx, a.key)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("volume id allocator read: %w", err)
|
||||
}
|
||||
var cur uint64
|
||||
cmp := clientv3.Compare(clientv3.CreateRevision(a.key), "=", 0)
|
||||
if len(resp.Kvs) > 0 {
|
||||
if cur, err = strconv.ParseUint(string(resp.Kvs[0].Value), 10, 32); err != nil {
|
||||
return 0, fmt.Errorf("volume id counter %q: %w", resp.Kvs[0].Value, err)
|
||||
}
|
||||
cmp = clientv3.Compare(clientv3.ModRevision(a.key), "=", resp.Kvs[0].ModRevision)
|
||||
}
|
||||
// A stale etcd counter (restored backup, re-enabled allocator) must
|
||||
// never go below the bound this master already knows. Resampled per
|
||||
// attempt because a heartbeat can raise it while the CAS retries.
|
||||
if seedBound := uint64(a.seed()); seedBound > cur {
|
||||
cur = seedBound
|
||||
}
|
||||
if cur >= math.MaxUint32 {
|
||||
return 0, fmt.Errorf("volume id counter exhausted at %d", cur)
|
||||
}
|
||||
committed, err := a.kv.Txn(ctx).If(cmp).Then(clientv3.OpPut(a.key, strconv.FormatUint(cur+1, 10))).Commit()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("volume id allocator commit: %w", err)
|
||||
}
|
||||
if !committed.Succeeded {
|
||||
continue
|
||||
}
|
||||
// The bound may have moved past the committed id while the CAS was in
|
||||
// flight. That id could name an existing volume, so skip it; wasting a
|
||||
// number is safe, reusing one is not.
|
||||
if latest := uint64(a.seed()); latest > cur {
|
||||
continue
|
||||
}
|
||||
return needle.VolumeId(cur + 1), nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
mvccpb "go.etcd.io/etcd/api/v3/mvccpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
// fakeEtcdKV is an in-memory clientv3.KV honoring CreateRevision/ModRevision
|
||||
// compare-and-swap, enough to exercise the allocator's CAS loop.
|
||||
type fakeEtcdKV struct {
|
||||
mu sync.Mutex
|
||||
rev int64
|
||||
data map[string]*mvccpb.KeyValue
|
||||
// beforeCmp runs inside Commit before the compares are evaluated, to
|
||||
// model a competing writer landing between the read and the CAS.
|
||||
beforeCmp func()
|
||||
getErr error
|
||||
commitErr error
|
||||
}
|
||||
|
||||
func newFakeEtcdKV() *fakeEtcdKV {
|
||||
return &fakeEtcdKV{data: make(map[string]*mvccpb.KeyValue)}
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
|
||||
kv.mu.Lock()
|
||||
defer kv.mu.Unlock()
|
||||
if kv.getErr != nil {
|
||||
return nil, kv.getErr
|
||||
}
|
||||
if v, ok := kv.data[key]; ok {
|
||||
cp := *v
|
||||
return &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{&cp}}, nil
|
||||
}
|
||||
return &clientv3.GetResponse{}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
|
||||
kv.mu.Lock()
|
||||
defer kv.mu.Unlock()
|
||||
kv.applyPut(key, val)
|
||||
return &clientv3.PutResponse{}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) applyPut(key, val string) {
|
||||
kv.rev++
|
||||
if v, ok := kv.data[key]; ok {
|
||||
v.Value = []byte(val)
|
||||
v.ModRevision = kv.rev
|
||||
} else {
|
||||
kv.data[key] = &mvccpb.KeyValue{Key: []byte(key), Value: []byte(val), CreateRevision: kv.rev, ModRevision: kv.rev}
|
||||
}
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) GetStream(ctx context.Context, key string, opts ...clientv3.OpOption) (clientv3.GetStreamChan, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
|
||||
kv.mu.Lock()
|
||||
defer kv.mu.Unlock()
|
||||
delete(kv.data, key)
|
||||
return &clientv3.DeleteResponse{}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
|
||||
return &clientv3.CompactResponse{}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
|
||||
return clientv3.OpResponse{}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) Txn(ctx context.Context) clientv3.Txn {
|
||||
return &fakeEtcdTxn{kv: kv}
|
||||
}
|
||||
|
||||
type fakeEtcdTxn struct {
|
||||
kv *fakeEtcdKV
|
||||
cmps []clientv3.Cmp
|
||||
ops []clientv3.Op
|
||||
}
|
||||
|
||||
func (t *fakeEtcdTxn) If(cs ...clientv3.Cmp) clientv3.Txn {
|
||||
t.cmps = append(t.cmps, cs...)
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *fakeEtcdTxn) Then(ops ...clientv3.Op) clientv3.Txn {
|
||||
t.ops = append(t.ops, ops...)
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *fakeEtcdTxn) Else(ops ...clientv3.Op) clientv3.Txn {
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *fakeEtcdTxn) Commit() (*clientv3.TxnResponse, error) {
|
||||
t.kv.mu.Lock()
|
||||
defer t.kv.mu.Unlock()
|
||||
if t.kv.beforeCmp != nil {
|
||||
t.kv.beforeCmp()
|
||||
}
|
||||
for _, cmp := range t.cmps {
|
||||
if !t.kv.evalCmp(cmp) {
|
||||
return &clientv3.TxnResponse{Succeeded: false}, nil
|
||||
}
|
||||
}
|
||||
for _, op := range t.ops {
|
||||
if op.IsPut() {
|
||||
t.kv.applyPut(string(op.KeyBytes()), string(op.ValueBytes()))
|
||||
}
|
||||
}
|
||||
if t.kv.commitErr != nil {
|
||||
return nil, t.kv.commitErr
|
||||
}
|
||||
return &clientv3.TxnResponse{Succeeded: true}, nil
|
||||
}
|
||||
|
||||
func (kv *fakeEtcdKV) evalCmp(cmp clientv3.Cmp) bool {
|
||||
c := cmp.GetCompare()
|
||||
if c.Result != etcdserverpb.Compare_EQUAL {
|
||||
return false
|
||||
}
|
||||
existing := kv.data[string(c.Key)]
|
||||
switch u := c.TargetUnion.(type) {
|
||||
case *etcdserverpb.Compare_CreateRevision:
|
||||
var rev int64
|
||||
if existing != nil {
|
||||
rev = existing.CreateRevision
|
||||
}
|
||||
return rev == u.CreateRevision
|
||||
case *etcdserverpb.Compare_ModRevision:
|
||||
var rev int64
|
||||
if existing != nil {
|
||||
rev = existing.ModRevision
|
||||
}
|
||||
return rev == u.ModRevision
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorSeedsFromLocalBound(t *testing.T) {
|
||||
alloc := newEtcdVolumeIdAllocator(newFakeEtcdKV(), "/sw/", time.Second, func() needle.VolumeId {
|
||||
return 41
|
||||
})
|
||||
for i, want := range []needle.VolumeId{42, 43, 44} {
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc %d: %v", i, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("alloc %d = %d, want %d", i, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorNeverBelowLocalBound(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
kv.Put(context.Background(), "seaweedfs/max_volume_id", "10")
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 41
|
||||
})
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 42 {
|
||||
t.Fatalf("a stale stored counter must not reissue ids: got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorResumesFromStoredKey(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
kv.Put(context.Background(), "seaweedfs/max_volume_id", "100")
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 41
|
||||
})
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 101 {
|
||||
t.Fatalf("stored counter wins over the seed: got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorRetriesOnCasConflict(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
// a competing writer lands the key once, between the read and the CAS
|
||||
kv.beforeCmp = func() {
|
||||
kv.applyPut("seaweedfs/max_volume_id", "55")
|
||||
kv.beforeCmp = nil
|
||||
}
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 5
|
||||
})
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 56 {
|
||||
t.Fatalf("CAS retry should allocate 56, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorSkipsIdBelowFreshBound(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
var bound atomic.Uint32
|
||||
bound.Store(5)
|
||||
// a heartbeat raises the local bound while the CAS is in flight; the
|
||||
// committed 6 must not be handed out
|
||||
kv.beforeCmp = func() {
|
||||
bound.Store(43)
|
||||
kv.beforeCmp = nil
|
||||
}
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return needle.VolumeId(bound.Load())
|
||||
})
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 44 {
|
||||
t.Fatalf("an id at or below the fresher bound must be skipped, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorReadError(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
kv.getErr = errors.New("etcd read failed")
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 41
|
||||
})
|
||||
if _, err := alloc.NextVolumeId(); err == nil {
|
||||
t.Fatal("a failed read must return an error, not an id")
|
||||
}
|
||||
kv.getErr = nil
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 42 {
|
||||
t.Fatalf("got %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorCommitError(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
kv.commitErr = errors.New("etcd commit failed")
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 41
|
||||
})
|
||||
if _, err := alloc.NextVolumeId(); err == nil {
|
||||
t.Fatal("a failed commit must return an error, not an id")
|
||||
}
|
||||
kv.commitErr = nil
|
||||
got, err := alloc.NextVolumeId()
|
||||
if err != nil {
|
||||
t.Fatalf("alloc: %v", err)
|
||||
}
|
||||
if got != 43 {
|
||||
t.Fatalf("the id written by the failed commit must be skipped, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcdAllocatorExhaustion(t *testing.T) {
|
||||
kv := newFakeEtcdKV()
|
||||
kv.Put(context.Background(), "seaweedfs/max_volume_id", strconv.FormatUint(math.MaxUint32, 10))
|
||||
alloc := newEtcdVolumeIdAllocator(kv, "seaweedfs", time.Second, func() needle.VolumeId {
|
||||
return 5
|
||||
})
|
||||
if _, err := alloc.NextVolumeId(); err == nil {
|
||||
t.Fatal("expected exhaustion error at MaxUint32")
|
||||
}
|
||||
}
|
||||
@@ -411,6 +411,9 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) {
|
||||
// or one on a server in maintenance mode, holds the whole volume out.
|
||||
func (vl *VolumeLayout) isAllWritable(vid needle.VolumeId) bool {
|
||||
if location, ok := vl.vid2location[vid]; ok {
|
||||
if location.AnyReadOnly() {
|
||||
return false
|
||||
}
|
||||
for _, dn := range location.list {
|
||||
if dn.InMaintenanceMode() {
|
||||
return false
|
||||
@@ -586,12 +589,21 @@ func (vl *VolumeLayout) isEmpty() bool {
|
||||
return len(vl.vid2location) == 0
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) ListVids() (vids []needle.VolumeId) {
|
||||
vl.accessLock.RLock()
|
||||
defer vl.accessLock.RUnlock()
|
||||
for vid := range vl.vid2location {
|
||||
vids = append(vids, vid)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) Lookup(vid needle.VolumeId) []*DataNode {
|
||||
vl.accessLock.RLock()
|
||||
defer vl.accessLock.RUnlock()
|
||||
|
||||
if location := vl.vid2location[vid]; location != nil {
|
||||
return location.list
|
||||
return append([]*DataNode(nil), location.list...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1021,6 +1033,19 @@ func (vl *VolumeLayout) SetVolumeWritable(dn *DataNode, vid needle.VolumeId) boo
|
||||
return false
|
||||
}
|
||||
|
||||
// SetVolumeQuarantined marks dn's replica read-only at the layout level only.
|
||||
// The node's stored report is left untouched so the volume digest still tracks
|
||||
// what the server reports and quarantined copies do not force endless full
|
||||
// heartbeats.
|
||||
func (vl *VolumeLayout) SetVolumeQuarantined(dn *DataNode, vid needle.VolumeId) {
|
||||
vl.accessLock.Lock()
|
||||
defer vl.accessLock.Unlock()
|
||||
if location, ok := vl.vid2location[vid]; ok {
|
||||
location.SetReadOnly(dn, true)
|
||||
}
|
||||
vl.removeFromWritable(vid)
|
||||
}
|
||||
|
||||
// SetReplicaReadOnlyFlag records what dn's latest heartbeat said about its
|
||||
// replica of vid, and nothing more: the writable list is left to
|
||||
// EnsureCorrectWritables and its capacity guards. Until now the flag only moved
|
||||
|
||||
Reference in new issue
Block a user