mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
feat: send EC shards in parallel per destination disk (#12050)
* feat: send EC shards in parallel per destination disk One stream per destination disk. max_shard_transfers unset means no global cap, 0 sends shards sequentially, and N is a global cap that still keeps one stream per disk. The value is persisted through TaskPolicy. The maintenance form is a checkbox plus Max transfers because unset and 0 differ. The key is intentionally omitted from GetConfigSpec. * ec shard distribution: treat negative max_shard_transfers as sequential The config layer (normalizedMaxShardTransfers) clamps negative values to 0, which selects sequential distribution. shardTransferCap diverged: a negative cap that reached ShardDistributionOptions directly would have selected uncapped parallel mode, the most aggressive setting. Align the options layer with the same normalization so invalid input degrades to the conservative mode. * ec: serialize shard transfers to a server when a target leaves disk choice to the receiver disk_id 0 is auto-placement on the receiving volume server, so an auto-placed stream can land on a disk another queued stream is pinned to. Grouping by (destNode, disk_id) alone let both run at once; collapse all of that server's transfers under the destNode key instead. * worker: clamp max_shard_transfers to the int32 wire range ReadIntConfig returned the 0 fallback for values above MaxInt32, turning a large positive cap into sequential mode. Read int64 and clamp into range instead. * worker: drop unused shard-transfer form helper The maintenance form renders and collects max_shard_transfers in plugin.templ JavaScript; the Go copy had no callers. * worker: bound max_shard_transfers conversion under an in-range guard --------- Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
15 files changed
+1089
-26
No files matched your search
@@ -2,6 +2,7 @@ package erasure_coding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -20,6 +21,9 @@ type Config struct {
|
||||
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
|
||||
StrictPlacement bool `json:"strict_placement"` // fail planning instead of relaxing placement constraints
|
||||
FreeSlotsDeadband int `json:"free_slots_deadband"` // Place volume-slot M; |ΔfreeSlots| ≤ M×dataShards is a tie, then liveShards; 0 = strict
|
||||
// MaxShardTransfers is the global in-flight cap for EC shard copies.
|
||||
// nil = no cap (one in-flight per dest disk); 0 = sequential; N>0 = cap.
|
||||
MaxShardTransfers *int `json:"max_shard_transfers,omitempty"`
|
||||
}
|
||||
|
||||
// NewDefaultConfig creates a new default erasure coding configuration
|
||||
@@ -224,11 +228,51 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
||||
ReplicaPlacement: c.ReplicaPlacement,
|
||||
StrictPlacement: c.StrictPlacement,
|
||||
FreeSlotsDeadband: int32(c.FreeSlotsDeadband),
|
||||
MaxShardTransfers: optionalInt32(c.MaxShardTransfers),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// optionalInt32 keeps unset (nil) distinct from sequential (0) on the proto field.
|
||||
func optionalInt32(n *int) *int32 {
|
||||
n = normalizedMaxShardTransfers(n)
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
v := int32(*n)
|
||||
return &v
|
||||
}
|
||||
|
||||
// normalizedMaxShardTransfers clamps a set cap into the proto int32 range.
|
||||
// nil stays nil so an absent knob is not stored as sequential.
|
||||
func normalizedMaxShardTransfers(n *int) *int {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
v := *n
|
||||
if v < 0 {
|
||||
v = 0
|
||||
}
|
||||
if v > math.MaxInt32 {
|
||||
v = math.MaxInt32
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
func optionalIntFromProto(n *int32) *int {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
v := int(*n)
|
||||
return &v
|
||||
}
|
||||
|
||||
// cloneMaxShardTransfers copies the cap so a task does not alias the live config.
|
||||
func cloneMaxShardTransfers(n *int) *int {
|
||||
return normalizedMaxShardTransfers(n)
|
||||
}
|
||||
|
||||
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
||||
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
if policy == nil {
|
||||
@@ -250,6 +294,8 @@ func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
c.ReplicaPlacement = ecConfig.ReplicaPlacement
|
||||
c.StrictPlacement = ecConfig.StrictPlacement
|
||||
c.FreeSlotsDeadband = int(ecConfig.FreeSlotsDeadband)
|
||||
// Read the pointer, not GetMaxShardTransfers: the getter maps unset to 0.
|
||||
c.MaxShardTransfers = normalizedMaxShardTransfers(optionalIntFromProto(ecConfig.MaxShardTransfers))
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestMaxShardTransfersPolicyRoundTrip(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in *int
|
||||
}{
|
||||
{name: "unset", in: nil},
|
||||
{name: "sequential", in: intPtr(0)},
|
||||
{name: "cap", in: intPtr(4)},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := NewDefaultConfig()
|
||||
cfg.MaxShardTransfers = cloneMaxShardTransfers(tc.in)
|
||||
|
||||
raw, err := proto.Marshal(cfg.ToTaskPolicy())
|
||||
require.NoError(t, err)
|
||||
var decoded worker_pb.TaskPolicy
|
||||
require.NoError(t, proto.Unmarshal(raw, &decoded))
|
||||
|
||||
restored := NewDefaultConfig()
|
||||
require.NoError(t, restored.FromTaskPolicy(&decoded))
|
||||
ec := decoded.GetErasureCodingConfig()
|
||||
require.NotNil(t, ec)
|
||||
if tc.in == nil {
|
||||
require.Nil(t, ec.MaxShardTransfers)
|
||||
require.Nil(t, restored.MaxShardTransfers)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, ec.MaxShardTransfers)
|
||||
require.Equal(t, int32(*tc.in), *ec.MaxShardTransfers)
|
||||
require.NotNil(t, restored.MaxShardTransfers)
|
||||
require.Equal(t, *tc.in, *restored.MaxShardTransfers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxShardTransfersNegativeClampsToSequential(t *testing.T) {
|
||||
cfg := NewDefaultConfig()
|
||||
cfg.MaxShardTransfers = intPtr(-3)
|
||||
restored := NewDefaultConfig()
|
||||
require.NoError(t, restored.FromTaskPolicy(cfg.ToTaskPolicy()))
|
||||
require.NotNil(t, restored.MaxShardTransfers)
|
||||
require.Equal(t, 0, *restored.MaxShardTransfers)
|
||||
}
|
||||
|
||||
func intPtr(n int) *int { return &n }
|
||||
@@ -64,6 +64,10 @@ type ErasureCodingTask struct {
|
||||
// with, read back off the EC context. Every holder must report serving the
|
||||
// same one before the source volume may be deleted.
|
||||
encodedBlockSize int64
|
||||
|
||||
// maxShardTransfers is the optional global cap for shard fan-out.
|
||||
// nil = no cap (one in-flight per dest disk/node); 0 = sequential; N>0 = cap.
|
||||
maxShardTransfers *int
|
||||
}
|
||||
|
||||
// NewErasureCodingTask creates a new unified EC task instance
|
||||
@@ -797,7 +801,9 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
|
||||
// distributeEcShards distributes locally generated EC shards to destination servers
|
||||
// using pre-assigned shard IDs from planning phase
|
||||
func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) error {
|
||||
assignment, err := erasure_coding.DistributeEcShards(t.volumeID, t.collection, t.targets, shardFiles, t.grpcDialOption, t.GetLogger())
|
||||
assignment, err := erasure_coding.DistributeEcShards(t.volumeID, t.collection, t.targets, shardFiles, t.grpcDialOption, t.GetLogger(), &erasure_coding.ShardDistributionOptions{
|
||||
MaxShardTransfers: t.maxShardTransfers,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package erasure_coding
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -174,6 +175,20 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
SectionId: "shard_distribution",
|
||||
Title: "Shard distribution",
|
||||
Description: "How many EC shard copies are in flight. The checkbox is presentation only; the saved value is max_shard_transfers.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "max_shard_transfers",
|
||||
Label: "Parallel shard distribution",
|
||||
Description: "Unchecked sends shards sequentially. Checked with an empty max uses one in-flight copy per destination disk. A number is a global cap, still one copy per disk.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"quiet_for_seconds": {
|
||||
@@ -545,6 +560,7 @@ func (h *ErasureCodingHandler) Execute(
|
||||
h.grpcDialOption,
|
||||
)
|
||||
task.workerAddress = h.workerAddress
|
||||
task.maxShardTransfers = deriveErasureCodingWorkerConfig(request.GetWorkerConfigValues()).TaskConfig.MaxShardTransfers
|
||||
execCtx, execCancel := context.WithCancel(ctx)
|
||||
defer execCancel()
|
||||
task.SetProgressCallback(func(progress float64, stage string) {
|
||||
@@ -669,12 +685,33 @@ func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *
|
||||
deadband = 100000
|
||||
}
|
||||
taskConfig.FreeSlotsDeadband = deadband
|
||||
taskConfig.MaxShardTransfers = maxShardTransfersFromValues(values)
|
||||
|
||||
return &erasureCodingWorkerConfig{
|
||||
TaskConfig: taskConfig,
|
||||
}
|
||||
}
|
||||
|
||||
// maxShardTransfersFromValues returns nil when the key is absent so shard
|
||||
// distribution keeps the uncapped default. 0 is sequential; N>0 is a global cap.
|
||||
func maxShardTransfersFromValues(values map[string]*plugin_pb.ConfigValue) *int {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := values["max_shard_transfers"]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
n := pluginworker.ReadInt64Config(values, "max_shard_transfers", 0)
|
||||
v := 0
|
||||
if n > math.MaxInt32 {
|
||||
v = math.MaxInt32
|
||||
} else if n > 0 {
|
||||
v = int(n)
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
func buildErasureCodingProposal(
|
||||
result *workertypes.TaskDetectionResult,
|
||||
baseWorkingDir string,
|
||||
|
||||
@@ -2,6 +2,7 @@ package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -139,6 +140,46 @@ func TestDeriveErasureCodingWorkerConfig(t *testing.T) {
|
||||
if cfg.TaskConfig.FreeSlotsDeadband != 12 {
|
||||
t.Fatalf("expected free_slots_deadband 12, got %d", cfg.TaskConfig.FreeSlotsDeadband)
|
||||
}
|
||||
if cfg.TaskConfig.MaxShardTransfers != nil {
|
||||
t.Fatalf("missing max_shard_transfers must stay unset, got %d", *cfg.TaskConfig.MaxShardTransfers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveErasureCodingWorkerConfigMaxShardTransfers(t *testing.T) {
|
||||
if got := deriveErasureCodingWorkerConfig(nil).TaskConfig.MaxShardTransfers; got != nil {
|
||||
t.Fatalf("nil values: got %d, want unset", *got)
|
||||
}
|
||||
if got := deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{}).TaskConfig.MaxShardTransfers; got != nil {
|
||||
t.Fatalf("missing key: got %d, want unset", *got)
|
||||
}
|
||||
|
||||
cfg := deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
|
||||
"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
})
|
||||
if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 0 {
|
||||
t.Fatalf("0: got %v, want 0 (sequential)", cfg.TaskConfig.MaxShardTransfers)
|
||||
}
|
||||
|
||||
cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
|
||||
"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 4}},
|
||||
})
|
||||
if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 4 {
|
||||
t.Fatalf("4: got %v, want 4", cfg.TaskConfig.MaxShardTransfers)
|
||||
}
|
||||
|
||||
cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
|
||||
"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -3}},
|
||||
})
|
||||
if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 0 {
|
||||
t.Fatalf("negative: got %v, want 0", cfg.TaskConfig.MaxShardTransfers)
|
||||
}
|
||||
|
||||
cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
|
||||
"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: math.MaxInt32 + 1}},
|
||||
})
|
||||
if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != math.MaxInt32 {
|
||||
t.Fatalf("above int32: got %v, want clamp to %d", cfg.TaskConfig.MaxShardTransfers, int64(math.MaxInt32))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildErasureCodingProposal(t *testing.T) {
|
||||
@@ -291,6 +332,15 @@ func TestErasureCodingDescriptorExposesFreeSlotsDeadband(t *testing.T) {
|
||||
if sectionID != "placement" {
|
||||
t.Fatalf("free_slots_deadband section = %q, want placement", sectionID)
|
||||
}
|
||||
if !workerConfigFormHasField(descriptor.WorkerConfigForm, "max_shard_transfers") {
|
||||
t.Fatal("worker config form omitted max_shard_transfers")
|
||||
}
|
||||
if _, ok := descriptor.WorkerConfigForm.DefaultValues["max_shard_transfers"]; ok {
|
||||
t.Fatal("max_shard_transfers default would collapse unset into 0")
|
||||
}
|
||||
if _, ok := descriptor.WorkerDefaultValues["max_shard_transfers"]; ok {
|
||||
t.Fatal("worker default max_shard_transfers would collapse unset into 0")
|
||||
}
|
||||
if prev != "placement" {
|
||||
t.Fatalf("replica_placement section = %q, want placement next to free_slots_deadband", prev)
|
||||
}
|
||||
|
||||
@@ -51,13 +51,17 @@ func RegisterErasureCodingTask() {
|
||||
if len(params.Sources) == 0 {
|
||||
return nil, fmt.Errorf("at least one source is required for erasure coding task")
|
||||
}
|
||||
return NewErasureCodingTask(
|
||||
task := NewErasureCodingTask(
|
||||
fmt.Sprintf("erasure_coding-%d", params.VolumeId),
|
||||
params.Sources[0].Node, // Use first source node
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
dialOpt,
|
||||
), nil
|
||||
)
|
||||
if cfg, ok := globalTaskDef.Config.(*Config); ok {
|
||||
task.maxShardTransfers = cloneMaxShardTransfers(cfg.MaxShardTransfers)
|
||||
}
|
||||
return task, nil
|
||||
},
|
||||
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
|
||||
|
||||
Reference in new issue
Block a user