mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
vacuum: bound concurrent commits per volume server (#11658)
CommitCompact holds the volume's write lock for the whole commit, and reads of that volume wait on it. Volumes compacted together tend to finish together, so their commits land at once and their fsyncs contend: each commit's lock hold grows with how many run alongside it, even when the disk is far from its throughput limit. Add -compactionMaxParallelCommits (volume.compactionMaxParallelCommits for server/mini). A commit waits for a slot before CommitCompactVolume, so a queued commit holds no volume lock and reads keep flowing. The wait counts against the caller's deadline and returns the context error if it expires; a slot granted to an already-expired context is handed back, so a request the caller gave up on never starts a commit, and maintenance mode is checked again once the slot is held. The slot is released whether the commit succeeds or fails. Compaction itself is not gated, so -maxParallelVacuumPerServer still sets reclaim throughput. 0, the default, keeps today's behaviour. The limit sits in the volume server so it covers every commit caller: the master's vacuum sweep, the vacuum worker task and volume.tier.compact. The Rust volume server mirrors the flag and behaviour: queued commits wait on a semaphore keyed to the grpc-timeout deadline, hold no store lock, recheck maintenance mode once a permit is held, and release it on every exit path.
This commit is contained in:
1 parent
42ff7f2b8e
commit
42ff535511
18 files changed
+641
No files matched your search
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
var numCPU = runtime.NumCPU()
|
||||
@@ -96,6 +97,16 @@ func (vs *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_serv
|
||||
return nil, err
|
||||
}
|
||||
|
||||
release, err := vs.acquireVacuumCommitSlot(ctx)
|
||||
if err != nil {
|
||||
glog.Warningf("commit volume %d not started: %v", req.VolumeId, err)
|
||||
return nil, status.FromContextError(err).Err()
|
||||
}
|
||||
defer release()
|
||||
if err := vs.CheckMaintenanceMode(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
stats.VolumeServerVacuumingHistogram.WithLabelValues("commit").Observe(time.Since(start).Seconds())
|
||||
@@ -117,6 +128,31 @@ func (vs *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_serv
|
||||
|
||||
}
|
||||
|
||||
// acquireVacuumCommitSlot waits for a commit slot holding no volume lock; the
|
||||
// wait counts against the caller's deadline.
|
||||
func (vs *VolumeServer) acquireVacuumCommitSlot(ctx context.Context) (release func(), err error) {
|
||||
if vs.vacuumCommitSlots == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
vs.vacuumCommitsWaiting.Add(1)
|
||||
select {
|
||||
case vs.vacuumCommitSlots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
vs.vacuumCommitsWaiting.Add(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
release = func() { <-vs.vacuumCommitSlots }
|
||||
// select picks randomly when the slot and the deadline are ready together.
|
||||
if err := ctx.Err(); err != nil {
|
||||
release()
|
||||
return nil, err
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {
|
||||
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type fakeVacuumCompactStream struct {
|
||||
grpc.ServerStream
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (s *fakeVacuumCompactStream) Context() context.Context { return s.ctx }
|
||||
|
||||
func (s *fakeVacuumCompactStream) Send(*volume_server_pb.VacuumVolumeCompactResponse) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newVacuumCommitTestServer(t *testing.T, maxParallelCommits int, vids ...needle.VolumeId) *VolumeServer {
|
||||
t.Helper()
|
||||
store := newTraversalTestStore(t.TempDir())
|
||||
t.Cleanup(store.Close)
|
||||
for _, vid := range vids {
|
||||
require.NoError(t, store.AddVolume(vid, "", storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
||||
for i := 1; i <= 3; i++ {
|
||||
n := new(needle.Needle)
|
||||
n.Id = types.Uint64ToNeedleId(uint64(i))
|
||||
n.Cookie = 0x1234
|
||||
n.Data = []byte(fmt.Sprintf("volume-%d-needle-%d", vid, i))
|
||||
n.Checksum = needle.NewCRC(n.Data)
|
||||
_, err := store.WriteVolumeNeedle(vid, n, true, false)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
vs := &VolumeServer{store: store}
|
||||
if maxParallelCommits > 0 {
|
||||
vs.vacuumCommitSlots = make(chan struct{}, maxParallelCommits)
|
||||
}
|
||||
return vs
|
||||
}
|
||||
|
||||
func compactForTest(vs *VolumeServer, vid needle.VolumeId) error {
|
||||
stream := &fakeVacuumCompactStream{ctx: context.Background()}
|
||||
return vs.VacuumVolumeCompact(&volume_server_pb.VacuumVolumeCompactRequest{VolumeId: uint32(vid)}, stream)
|
||||
}
|
||||
|
||||
func maxConcurrentHolders(vs *VolumeServer, workers int) int32 {
|
||||
var current, peak int32
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
release, err := vs.acquireVacuumCommitSlot(context.Background())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
n := atomic.AddInt32(¤t, 1)
|
||||
for {
|
||||
p := atomic.LoadInt32(&peak)
|
||||
if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
atomic.AddInt32(¤t, -1)
|
||||
release()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return atomic.LoadInt32(&peak)
|
||||
}
|
||||
|
||||
func TestVacuumCommitSlotsBoundConcurrentCommits(t *testing.T) {
|
||||
assert.Equal(t, int32(1), maxConcurrentHolders(&VolumeServer{vacuumCommitSlots: make(chan struct{}, 1)}, 8))
|
||||
assert.LessOrEqual(t, maxConcurrentHolders(&VolumeServer{vacuumCommitSlots: make(chan struct{}, 3)}, 8), int32(3))
|
||||
}
|
||||
|
||||
func TestVacuumCommitSlotsUnlimitedByDefault(t *testing.T) {
|
||||
vs := &VolumeServer{}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
for i := 0; i < 8; i++ {
|
||||
_, err := vs.acquireVacuumCommitSlot(ctx)
|
||||
require.NoError(t, err, "acquisition %d must not wait when no limit is configured", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumCommitSlotNotGrantedToExpiredContext(t *testing.T) {
|
||||
vs := &VolumeServer{vacuumCommitSlots: make(chan struct{}, 1)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for i := 0; i < 100; i++ {
|
||||
release, err := vs.acquireVacuumCommitSlot(ctx)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Nil(t, release)
|
||||
require.Empty(t, vs.vacuumCommitSlots, "a refused acquisition must not keep the slot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumCommitLimitLeavesCompactionAndOtherServersAlone(t *testing.T) {
|
||||
vids := []needle.VolumeId{1, 2, 3, 4}
|
||||
vs := newVacuumCommitTestServer(t, 1, vids...)
|
||||
other := newVacuumCommitTestServer(t, 1, 9)
|
||||
vs.vacuumCommitSlots <- struct{}{}
|
||||
|
||||
errs := make(chan error, len(vids))
|
||||
for _, vid := range vids {
|
||||
go func(vid needle.VolumeId) {
|
||||
errs <- compactForTest(vs, vid)
|
||||
}(vid)
|
||||
}
|
||||
for range vids {
|
||||
require.NoError(t, <-errs)
|
||||
}
|
||||
|
||||
require.NoError(t, compactForTest(other, 9))
|
||||
_, err := other.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 9})
|
||||
require.NoError(t, err, "a busy commit slot on one server must not block another server")
|
||||
}
|
||||
|
||||
func TestQueuedVacuumCommitHoldsNoVolumeLockAndHonoursDeadline(t *testing.T) {
|
||||
vid := needle.VolumeId(1)
|
||||
vs := newVacuumCommitTestServer(t, 1, vid)
|
||||
require.NoError(t, compactForTest(vs, vid))
|
||||
vs.vacuumCommitSlots <- struct{}{}
|
||||
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
|
||||
committed <- err
|
||||
}()
|
||||
require.Eventually(t, func() bool { return vs.vacuumCommitsWaiting.Load() == 1 }, 5*time.Second, time.Millisecond,
|
||||
"the commit must pass the entry checks and wait for a slot")
|
||||
|
||||
readDone := make(chan error, 1)
|
||||
go func() {
|
||||
n := &needle.Needle{Id: types.Uint64ToNeedleId(1), Cookie: 0x1234}
|
||||
_, err := vs.store.ReadVolumeNeedle(vid, n, nil, nil)
|
||||
readDone <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-readDone:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("read blocked behind a queued commit")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := vs.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
|
||||
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
|
||||
|
||||
select {
|
||||
case err := <-committed:
|
||||
t.Fatalf("commit ran while the only slot was taken: %v", err)
|
||||
default:
|
||||
}
|
||||
|
||||
<-vs.vacuumCommitSlots
|
||||
select {
|
||||
case err := <-committed:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("queued commit did not run after the slot freed")
|
||||
}
|
||||
assert.Empty(t, vs.vacuumCommitSlots, "the commit must return its slot")
|
||||
}
|
||||
|
||||
func TestVacuumCommitReleasesSlotOnError(t *testing.T) {
|
||||
vs := newVacuumCommitTestServer(t, 1)
|
||||
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 42})
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, vs.vacuumCommitSlots, "a failed commit must return its slot")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
release, err := vs.acquireVacuumCommitSlot(ctx)
|
||||
require.NoError(t, err)
|
||||
release()
|
||||
}
|
||||
|
||||
func TestQueuedVacuumCommitDoesNotStartInMaintenanceMode(t *testing.T) {
|
||||
vid := needle.VolumeId(1)
|
||||
vs := newVacuumCommitTestServer(t, 1, vid)
|
||||
require.NoError(t, compactForTest(vs, vid))
|
||||
revisionBefore := vs.store.GetVolume(vid).SuperBlock.CompactionRevision
|
||||
vs.vacuumCommitSlots <- struct{}{}
|
||||
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
|
||||
committed <- err
|
||||
}()
|
||||
require.Eventually(t, func() bool { return vs.vacuumCommitsWaiting.Load() == 1 }, 5*time.Second, time.Millisecond,
|
||||
"the commit must pass the entry maintenance check and wait for a slot")
|
||||
|
||||
require.NoError(t, vs.store.State.Update(&volume_server_pb.VolumeServerState{Maintenance: true}))
|
||||
<-vs.vacuumCommitSlots
|
||||
|
||||
select {
|
||||
case err := <-committed:
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "maintenance mode")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("queued commit did not return after the slot freed")
|
||||
}
|
||||
assert.Equal(t, revisionBefore, vs.store.GetVolume(vid).SuperBlock.CompactionRevision, "the volume must not be committed")
|
||||
assert.Empty(t, vs.vacuumCommitSlots, "the refused commit must return its slot")
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -55,6 +56,8 @@ type VolumeServer struct {
|
||||
AllowUntrustedRemoteEndpoints bool
|
||||
compactionBytePerSecond int64
|
||||
maintenanceBytePerSecond int64
|
||||
vacuumCommitSlots chan struct{}
|
||||
vacuumCommitsWaiting atomic.Int32
|
||||
metricsAddress string
|
||||
metricsIntervalSec int
|
||||
fileSizeLimitBytes int64
|
||||
@@ -79,6 +82,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
readMode string,
|
||||
compactionMBPerSecond int,
|
||||
maintenanceMBPerSecond int,
|
||||
maxParallelVacuumCommits int,
|
||||
fileSizeLimitMB int,
|
||||
concurrentUploadLimit int64,
|
||||
concurrentDownloadLimit int64,
|
||||
@@ -126,6 +130,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
||||
whiteList: whiteList,
|
||||
AllowUntrustedRemoteEndpoints: allowUntrustedRemoteEndpoints,
|
||||
}
|
||||
if maxParallelVacuumCommits > 0 {
|
||||
vs.vacuumCommitSlots = make(chan struct{}, maxParallelVacuumCommits)
|
||||
}
|
||||
|
||||
whiteList = append(whiteList, util.StringSplit(v.GetString("guard.white_list"), ",")...)
|
||||
// Copy the caller's slice so subsequent external mutation cannot desync
|
||||
|
||||
Reference in new issue
Block a user