seaweedfs/weed/storage/store_vacuum.go

package storage

import (
	"fmt"

	"github.com/seaweedfs/seaweedfs/weed/stats"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

var ErrInsufficientSpace = fmt.Errorf("insufficient free space")

func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (garbageRatio float64, diskSpaceLow bool, err error) {
	if v := s.findVolume(volumeId); v != nil {
		glog.V(3).Infof("volume %d garbage level: %f", volumeId, v.garbageLevel())
		// diskSpaceLow only counts when it is the sole read-only cause — an
		// operator mark or I/O quarantine still shields the volume.
		_, noWriteOrDelete, noWriteCanDelete, isLow := v.ReadOnlyReasons()
		return v.garbageLevel(), isLow && !noWriteOrDelete && !noWriteCanDelete, nil
	}
	return 0, false, fmt.Errorf("volume id %d is not found during check compact: %w", volumeId, ErrVolumeNotFound)
}

func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
	if v := s.findVolume(vid); v != nil {
		if err := ensureCompactVolumeSpace(v, preallocate); err != nil {
			return err
		}
		return v.CompactByIndex(&CompactOptions{
			PreallocateBytes:  preallocate,
			MaxBytesPerSecond: compactionBytePerSecond,
			ProgressCallback:  progressFn,
		})
	}
	return fmt.Errorf("volume id %d is not found during compact: %w", vid, ErrVolumeNotFound)
}

func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, int64, error) {
	if s.isStopping.Load() {
		return false, 0, fmt.Errorf("volume id %d skips compact because volume is stopping", vid)
	}
	if v := s.findVolume(vid); v != nil {
		isReadOnly := v.IsReadOnly()
		err := v.CommitCompact()
		var volumeSize int64 = 0
		if err == nil && v.DataBackend != nil {
			volumeSize, _, _ = v.DataBackend.GetStat()
		}
		return isReadOnly, volumeSize, err
	}
	return false, 0, fmt.Errorf("volume id %d is not found during commit compact: %w", vid, ErrVolumeNotFound)
}

func (s *Store) CommitCleanupVolume(vid needle.VolumeId) error {
	if v := s.findVolume(vid); v != nil {
		return v.cleanupCompact()
	}
	return fmt.Errorf("volume id %d is not found during cleaning up: %w", vid, ErrVolumeNotFound)
}

// compactionDiskFree reports the free bytes on the disk holding dir, and
// compactionSameFilesystem whether two directories draw on the same pool.
// Both are variables so a test can stand in for a full or a split disk.
var (
	compactionDiskFree = func(dir string) uint64 {
		return stats.NewDiskStatus(dir).Free
	}
	compactionSameFilesystem = sameFilesystem
)

// compactionSpaceNeeded estimates what CompactByIndex will write for v: a new
// .dat holding the live needles with their on-disk framing behind a superblock
// (or preallocate, when that is larger, since the file is preallocated to it),
// and a rebuilt index with one entry per live needle. The volume's current size
// is the wrong yardstick: the more garbage a volume holds, the less its
// compaction writes, and a store that filled up until its volumes went
// read-only is exactly where the all-garbage volumes must still compact to
// give the space back (issue #11516). Neither estimate exceeds the current file.
func compactionSpaceNeeded(v *Volume, preallocate int64) (dataBytes, indexBytes int64) {
	datSize, idxSize, _ := v.FileStat()
	dataBytes, indexBytes = int64(datSize), int64(idxSize)

	liveCount := int64(v.FileCount()) - int64(v.DeletedCount())
	liveContent := int64(v.ContentSize()) - int64(v.DeletedSize())
	// A .sdx converted back to .idx carries no deleted sizes (see
	// garbageLevel), and counters that disagree mean the metric is off;
	// either way the whole volume stays the estimate.
	deletedSizeKnown := v.DeletedCount() == 0 || v.DeletedSize() > 0
	if deletedSizeKnown && liveCount >= 0 && liveContent >= 0 {
		// GetActualSize(0) is the framing of an empty needle; another
		// padding unit covers the worst case for any other size.
		perNeedle := needle.GetActualSize(0, v.Version()) + types.NeedlePaddingSize
		// Counters rebuilt from an index file (LevelDB and sorted maps) go
		// through a Bloom filter with a 0.1% false positive rate that can
		// count a live needle as deleted. A few percent of headroom covers
		// that many times over.
		if estimate := withHeadroom(super_block.SuperBlockSize + liveContent + liveCount*perNeedle); estimate < dataBytes {
			dataBytes = estimate
		}
		if estimate := withHeadroom(liveCount * types.NeedleMapEntrySize); estimate < indexBytes {
			indexBytes = estimate
		}
	}
	if preallocate > dataBytes {
		dataBytes = preallocate
	}
	return dataBytes, indexBytes
}

func withHeadroom(estimate int64) int64 {
	return estimate + estimate/16
}

func ensureCompactVolumeSpace(v *Volume, preallocate int64) error {
	dataBytes, indexBytes := compactionSpaceNeeded(v, preallocate)
	volumeSize, indexSize, _ := v.FileStat()
	check := func(dir string, needed int64) error {
		free := compactionDiskFree(dir)
		if int64(free) < needed {
			return fmt.Errorf("insufficient free space for compaction in %s: need %d bytes (data: %d, index: %d, current volume: %d, current index: %d), but only %d bytes available: %w",
				dir, needed, dataBytes, indexBytes, volumeSize, indexSize, free, ErrInsufficientSpace)
		}
		glog.V(1).Infof("volume %d compaction space check in %s: data=%d, index=%d, current volume=%d, space_needed=%d, free_space=%d",
			v.Id, dir, dataBytes, indexBytes, volumeSize, needed, free)
		return nil
	}
	// The new .dat lands next to the old one and the new .idx next to the old
	// index. When the index directory is on another filesystem each disk
	// answers for its own share; two directories on one filesystem draw on
	// the same free space and must cover the sum.
	if v.dirIdx == "" || v.dirIdx == v.dir {
		return check(v.dir, dataBytes+indexBytes)
	}
	if !compactionSameFilesystem(v.dir, v.dirIdx) {
		if err := check(v.dir, dataBytes); err != nil {
			return err
		}
		return check(v.dirIdx, indexBytes)
	}
	// Same filesystem as far as the identity check can tell. The index
	// directory is still asked for its own share, because a mount point the
	// check cannot see (a volume mounted under one drive letter on Windows)
	// would otherwise go unchecked.
	if err := check(v.dir, dataBytes+indexBytes); err != nil {
		return err
	}
	return check(v.dirIdx, indexBytes)
}

func (s *Store) CompactVolumeFiles(vid needle.VolumeId, collection string, location *DiskLocation, needleMapKind NeedleMapKind, ldbTimeout int64, preallocate int64, compactionBytePerSecond int64) (err error) {
	if location == nil {
		return fmt.Errorf("volume %d compaction location is nil", vid)
	}

	tempVolume, err := loadVolumeWithoutWorker(location.Directory, location.IdxDirectory, collection, vid, needleMapKind, ldbTimeout)
	if err != nil {
		return fmt.Errorf("load volume %d for offline compaction: %w", vid, err)
	}
	tempVolume.location = location

	defer func() {
		if tempVolume.tmpNm != nil {
			tempVolume.tmpNm.Close()
			tempVolume.tmpNm = nil
		}
		tempVolume.doClose()
	}()

	if err := ensureCompactVolumeSpace(tempVolume, preallocate); err != nil {
		return err
	}

	if err := tempVolume.CompactByIndex(&CompactOptions{
		PreallocateBytes:  preallocate,
		MaxBytesPerSecond: compactionBytePerSecond,
	}); err != nil {
		if cleanupErr := tempVolume.cleanupCompact(); cleanupErr != nil {
			return fmt.Errorf("compact volume %d: %v (cleanup failed: %v)", vid, err, cleanupErr)
		}
		return fmt.Errorf("compact volume %d: %w", vid, err)
	}

	if err := tempVolume.CommitCompact(); err != nil {
		// A failed commit may have swapped only one of .dat/.idx; reconcile
		// rolls a decided swap forward or removes orphan temp files before
		// this volume can mount with a mismatched pair.
		if reconcileErr := tempVolume.reconcileCompactState(); reconcileErr != nil {
			return fmt.Errorf("commit compact volume %d: %v (reconcile failed: %v)", vid, err, reconcileErr)
		}
		if cleanupErr := tempVolume.cleanupCompact(); cleanupErr != nil {
			return fmt.Errorf("commit compact volume %d: %v (cleanup failed: %v)", vid, err, cleanupErr)
		}
		return fmt.Errorf("commit compact volume %d: %w", vid, err)
	}

	return nil
}