vacuum: bound concurrent commits per volume server (#11658)

CommitCompact holds the volume's write lock for the whole commit, and
reads of that volume wait on it. Volumes compacted together tend to
finish together, so their commits land at once and their fsyncs
contend: each commit's lock hold grows with how many run alongside it,
even when the disk is far from its throughput limit.

Add -compactionMaxParallelCommits (volume.compactionMaxParallelCommits
for server/mini). A commit waits for a slot before CommitCompactVolume,
so a queued commit holds no volume lock and reads keep flowing. The
wait counts against the caller's deadline and returns the context
error if it expires; a slot granted to an already-expired context is
handed back, so a request the caller gave up on never starts a commit,
and maintenance mode is checked again once the slot is held.
The slot is released whether the commit succeeds or fails. Compaction
itself is not gated, so -maxParallelVacuumPerServer still sets reclaim
throughput. 0, the default, keeps today's behaviour.

The limit sits in the volume server so it covers every commit caller:
the master's vacuum sweep, the vacuum worker task and volume.tier.compact.

The Rust volume server mirrors the flag and behaviour: queued commits
wait on a semaphore keyed to the grpc-timeout deadline, hold no store
lock, recheck maintenance mode once a permit is held, and release it
on every exit path.
This commit is contained in:
Peter Dodd authored and GitHub committed 2026-10-16 23:46:36 -07:00
1 parent 42ff7f2b8e
commit 42ff535511
18 files changed
+641

No files matched your search

+36
View File
@@ -14,6 +14,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"google.golang.org/grpc/status"
)
var numCPU = runtime.NumCPU()
@@ -96,6 +97,16 @@ func (vs *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_serv
return nil, err
}
release, err := vs.acquireVacuumCommitSlot(ctx)
if err != nil {
glog.Warningf("commit volume %d not started: %v", req.VolumeId, err)
return nil, status.FromContextError(err).Err()
}
defer release()
if err := vs.CheckMaintenanceMode(); err != nil {
return nil, err
}
start := time.Now()
defer func(start time.Time) {
stats.VolumeServerVacuumingHistogram.WithLabelValues("commit").Observe(time.Since(start).Seconds())
@@ -117,6 +128,31 @@ func (vs *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_serv
}
// acquireVacuumCommitSlot waits for a commit slot holding no volume lock; the
// wait counts against the caller's deadline.
func (vs *VolumeServer) acquireVacuumCommitSlot(ctx context.Context) (release func(), err error) {
if vs.vacuumCommitSlots == nil {
return func() {}, nil
}
vs.vacuumCommitsWaiting.Add(1)
select {
case vs.vacuumCommitSlots <- struct{}{}:
case <-ctx.Done():
err = ctx.Err()
}
vs.vacuumCommitsWaiting.Add(-1)
if err != nil {
return nil, err
}
release = func() { <-vs.vacuumCommitSlots }
// select picks randomly when the slot and the deadline are ready together.
if err := ctx.Err(); err != nil {
release()
return nil, err
}
return release, nil
}
func (vs *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
return nil, err
+225
View File
@@ -0,0 +1,225 @@
package weed_server
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type fakeVacuumCompactStream struct {
grpc.ServerStream
ctx context.Context
}
func (s *fakeVacuumCompactStream) Context() context.Context { return s.ctx }
func (s *fakeVacuumCompactStream) Send(*volume_server_pb.VacuumVolumeCompactResponse) error {
return nil
}
func newVacuumCommitTestServer(t *testing.T, maxParallelCommits int, vids ...needle.VolumeId) *VolumeServer {
t.Helper()
store := newTraversalTestStore(t.TempDir())
t.Cleanup(store.Close)
for _, vid := range vids {
require.NoError(t, store.AddVolume(vid, "", storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
for i := 1; i <= 3; i++ {
n := new(needle.Needle)
n.Id = types.Uint64ToNeedleId(uint64(i))
n.Cookie = 0x1234
n.Data = []byte(fmt.Sprintf("volume-%d-needle-%d", vid, i))
n.Checksum = needle.NewCRC(n.Data)
_, err := store.WriteVolumeNeedle(vid, n, true, false)
require.NoError(t, err)
}
}
vs := &VolumeServer{store: store}
if maxParallelCommits > 0 {
vs.vacuumCommitSlots = make(chan struct{}, maxParallelCommits)
}
return vs
}
func compactForTest(vs *VolumeServer, vid needle.VolumeId) error {
stream := &fakeVacuumCompactStream{ctx: context.Background()}
return vs.VacuumVolumeCompact(&volume_server_pb.VacuumVolumeCompactRequest{VolumeId: uint32(vid)}, stream)
}
func maxConcurrentHolders(vs *VolumeServer, workers int) int32 {
var current, peak int32
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
release, err := vs.acquireVacuumCommitSlot(context.Background())
if err != nil {
return
}
n := atomic.AddInt32(&current, 1)
for {
p := atomic.LoadInt32(&peak)
if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) {
break
}
}
time.Sleep(5 * time.Millisecond)
atomic.AddInt32(&current, -1)
release()
}()
}
wg.Wait()
return atomic.LoadInt32(&peak)
}
func TestVacuumCommitSlotsBoundConcurrentCommits(t *testing.T) {
assert.Equal(t, int32(1), maxConcurrentHolders(&VolumeServer{vacuumCommitSlots: make(chan struct{}, 1)}, 8))
assert.LessOrEqual(t, maxConcurrentHolders(&VolumeServer{vacuumCommitSlots: make(chan struct{}, 3)}, 8), int32(3))
}
func TestVacuumCommitSlotsUnlimitedByDefault(t *testing.T) {
vs := &VolumeServer{}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
for i := 0; i < 8; i++ {
_, err := vs.acquireVacuumCommitSlot(ctx)
require.NoError(t, err, "acquisition %d must not wait when no limit is configured", i)
}
}
func TestVacuumCommitSlotNotGrantedToExpiredContext(t *testing.T) {
vs := &VolumeServer{vacuumCommitSlots: make(chan struct{}, 1)}
ctx, cancel := context.WithCancel(context.Background())
cancel()
for i := 0; i < 100; i++ {
release, err := vs.acquireVacuumCommitSlot(ctx)
require.ErrorIs(t, err, context.Canceled)
require.Nil(t, release)
require.Empty(t, vs.vacuumCommitSlots, "a refused acquisition must not keep the slot")
}
}
func TestVacuumCommitLimitLeavesCompactionAndOtherServersAlone(t *testing.T) {
vids := []needle.VolumeId{1, 2, 3, 4}
vs := newVacuumCommitTestServer(t, 1, vids...)
other := newVacuumCommitTestServer(t, 1, 9)
vs.vacuumCommitSlots <- struct{}{}
errs := make(chan error, len(vids))
for _, vid := range vids {
go func(vid needle.VolumeId) {
errs <- compactForTest(vs, vid)
}(vid)
}
for range vids {
require.NoError(t, <-errs)
}
require.NoError(t, compactForTest(other, 9))
_, err := other.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 9})
require.NoError(t, err, "a busy commit slot on one server must not block another server")
}
func TestQueuedVacuumCommitHoldsNoVolumeLockAndHonoursDeadline(t *testing.T) {
vid := needle.VolumeId(1)
vs := newVacuumCommitTestServer(t, 1, vid)
require.NoError(t, compactForTest(vs, vid))
vs.vacuumCommitSlots <- struct{}{}
committed := make(chan error, 1)
go func() {
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
committed <- err
}()
require.Eventually(t, func() bool { return vs.vacuumCommitsWaiting.Load() == 1 }, 5*time.Second, time.Millisecond,
"the commit must pass the entry checks and wait for a slot")
readDone := make(chan error, 1)
go func() {
n := &needle.Needle{Id: types.Uint64ToNeedleId(1), Cookie: 0x1234}
_, err := vs.store.ReadVolumeNeedle(vid, n, nil, nil)
readDone <- err
}()
select {
case err := <-readDone:
require.NoError(t, err)
case <-time.After(5 * time.Second):
t.Fatal("read blocked behind a queued commit")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
_, err := vs.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
select {
case err := <-committed:
t.Fatalf("commit ran while the only slot was taken: %v", err)
default:
}
<-vs.vacuumCommitSlots
select {
case err := <-committed:
require.NoError(t, err)
case <-time.After(5 * time.Second):
t.Fatal("queued commit did not run after the slot freed")
}
assert.Empty(t, vs.vacuumCommitSlots, "the commit must return its slot")
}
func TestVacuumCommitReleasesSlotOnError(t *testing.T) {
vs := newVacuumCommitTestServer(t, 1)
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 42})
require.Error(t, err)
assert.Empty(t, vs.vacuumCommitSlots, "a failed commit must return its slot")
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
release, err := vs.acquireVacuumCommitSlot(ctx)
require.NoError(t, err)
release()
}
func TestQueuedVacuumCommitDoesNotStartInMaintenanceMode(t *testing.T) {
vid := needle.VolumeId(1)
vs := newVacuumCommitTestServer(t, 1, vid)
require.NoError(t, compactForTest(vs, vid))
revisionBefore := vs.store.GetVolume(vid).SuperBlock.CompactionRevision
vs.vacuumCommitSlots <- struct{}{}
committed := make(chan error, 1)
go func() {
_, err := vs.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: uint32(vid)})
committed <- err
}()
require.Eventually(t, func() bool { return vs.vacuumCommitsWaiting.Load() == 1 }, 5*time.Second, time.Millisecond,
"the commit must pass the entry maintenance check and wait for a slot")
require.NoError(t, vs.store.State.Update(&volume_server_pb.VolumeServerState{Maintenance: true}))
<-vs.vacuumCommitSlots
select {
case err := <-committed:
require.Error(t, err)
assert.Contains(t, err.Error(), "maintenance mode")
case <-time.After(5 * time.Second):
t.Fatal("queued commit did not return after the slot freed")
}
assert.Equal(t, revisionBefore, vs.store.GetVolume(vid).SuperBlock.CompactionRevision, "the volume must not be committed")
assert.Empty(t, vs.vacuumCommitSlots, "the refused commit must return its slot")
}
+7
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -55,6 +56,8 @@ type VolumeServer struct {
AllowUntrustedRemoteEndpoints bool
compactionBytePerSecond int64
maintenanceBytePerSecond int64
vacuumCommitSlots chan struct{}
vacuumCommitsWaiting atomic.Int32
metricsAddress string
metricsIntervalSec int
fileSizeLimitBytes int64
@@ -79,6 +82,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
readMode string,
compactionMBPerSecond int,
maintenanceMBPerSecond int,
maxParallelVacuumCommits int,
fileSizeLimitMB int,
concurrentUploadLimit int64,
concurrentDownloadLimit int64,
@@ -126,6 +130,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
whiteList: whiteList,
AllowUntrustedRemoteEndpoints: allowUntrustedRemoteEndpoints,
}
if maxParallelVacuumCommits > 0 {
vs.vacuumCommitSlots = make(chan struct{}, maxParallelVacuumCommits)
}
whiteList = append(whiteList, util.StringSplit(v.GetString("guard.white_list"), ",")...)
// Copy the caller's slice so subsequent external mutation cannot desync