From e6454498724f5f99b293d3eb2b27f1065923b099 Mon Sep 17 00:00:00 2001 From: Junker der Provinz Date: Thu, 22 Oct 2026 00:11:06 +0200 Subject: [PATCH] fix(storage): count preallocated space when auto-sizing max volumes (#11919) The max volume count subtracted each volume's remaining room from the free disk space, except when the master had preallocation turned on, in which case it subtracted nothing. That is wrong for a disk that mixes preallocated and plain volumes, and on platforms where preallocation is not supported. Use the larger of the .dat file's size and its allocated blocks, in the Go and the Rust volume server, so a preallocated volume counts as full and a plain one keeps its remaining room reserved. Fixes #5778 Co-authored-by: Dan Foster <4570306+danfoster@users.noreply.github.com> --- seaweed-volume/src/storage/disk_location.rs | 6 +- seaweed-volume/src/storage/store.rs | 95 +++++++++++-------- seaweed-volume/src/storage/volume.rs | 19 ++++ weed/storage/disk_location.go | 2 + weed/storage/disk_location_linux_test.go | 53 +++++++++++ weed/storage/store.go | 8 +- .../store_maxvolume_preallocate_test.go | 33 +++++++ weed/storage/volume.go | 16 ++++ weed/storage/volume_allocated_unix.go | 18 ++++ weed/storage/volume_allocated_windows.go | 10 ++ 10 files changed, 214 insertions(+), 46 deletions(-) create mode 100644 weed/storage/disk_location_linux_test.go create mode 100644 weed/storage/store_maxvolume_preallocate_test.go create mode 100644 weed/storage/volume_allocated_unix.go create mode 100644 weed/storage/volume_allocated_windows.go diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index 3d633e72aa..5483274f9c 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -740,7 +740,11 @@ impl DiskLocation { if vol.is_read_only() { continue; } - let dat_size = vol.dat_file_size().unwrap_or(0); + // A preallocated volume already holds its space on disk. + let dat_size = vol + .dat_file_size() + .unwrap_or(0) + .max(vol.dat_allocated_size()); let idx_size = vol.idx_file_size(); let used = dat_size + idx_size; if volume_size_limit > used { diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 014705c9f7..e1e12fb608 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -943,11 +943,7 @@ impl Store { let current = loc.max_volume_count.load(Ordering::Relaxed); let (_, free) = super::disk_location::get_disk_stats(&loc.directory); - let unused_space = if self.get_preallocate() { - 0 - } else { - loc.unused_space(volume_size_limit) - }; + let unused_space = loc.unused_space(volume_size_limit); let unclaimed = (free as i64) - (unused_space as i64); let vol_count = loc.volumes_len() as i32; @@ -2479,7 +2475,7 @@ mod tests { } #[test] - fn test_maybe_adjust_volume_max_honors_preallocate_flag() { + fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() { let tmp = TempDir::new().unwrap(); let dir = tmp.path().to_str().unwrap(); @@ -2494,13 +2490,6 @@ mod tests { Vec::new(), ) .unwrap(); - // Use a large volume_size_limit so the unused-space difference between - // preallocate=true (0) and preallocate=false (~2 × limit) is big enough - // that integer-division rounding and disk-free fluctuations between the - // two get_disk_stats calls cannot make the quotients equal. - store - .volume_size_limit - .store(100 * 1024 * 1024, Ordering::Relaxed); store .add_volume( VolumeId(61), @@ -2511,39 +2500,67 @@ mod tests { }, ) .unwrap(); + + // Room for 2.5 volumes: the plain volume claims one, leaving one more slot. + let (_, free) = crate::storage::disk_location::get_disk_stats(dir); + let volume_size_limit = (free as f64 / 2.5) as u64; + if volume_size_limit == 0 { + return; // cannot determine free disk space + } store - .add_volume( - VolumeId(62), + .volume_size_limit + .store(volume_size_limit, Ordering::Relaxed); + store.locations[0].original_max_volume_count = 0; + store.set_preallocate(true); + + store.maybe_adjust_volume_max(); + assert_eq!( + store.locations[0].max_volume_count.load(Ordering::Relaxed), + 2 + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn test_unused_space_counts_preallocated_volumes_as_full() { + const LIMIT: u64 = 1 << 20; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + let mut store = Store::new(NeedleMapKind::InMemory); + store + .add_location( + dir, + dir, + 2, DiskType::HardDrive, - &VolumeSpec { - collection: "preallocate_case", - ..Default::default() - }, + MinFreeSpace::Percent(1.0), + Vec::new(), ) .unwrap(); - for vid in [VolumeId(61), VolumeId(62)] { - let dat_path = store.find_volume(vid).unwrap().1.dat_path(); - std::fs::OpenOptions::new() - .write(true) - .open(dat_path) - .unwrap() - .set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64) + for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] { + store + .add_volume( + vid, + DiskType::HardDrive, + &VolumeSpec { + collection: "preallocate_case", + preallocate, + ..Default::default() + }, + ) .unwrap(); } - store.locations[0].original_max_volume_count = 0; - store.locations[0] - .max_volume_count - .store(0, Ordering::Relaxed); + let preallocated = store.find_volume(VolumeId(71)).unwrap().1; + if preallocated.dat_allocated_size() < LIMIT { + return; // the filesystem does not support fallocate + } - store.set_preallocate(false); - assert!(store.maybe_adjust_volume_max()); - let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed); - - store.set_preallocate(true); - assert!(store.maybe_adjust_volume_max()); - let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed); - - assert!(with_preallocate > without_preallocate); + let plain = store.find_volume(VolumeId(72)).unwrap().1; + let plain_dat = plain.dat_file_size().unwrap(); + // The filesystem may round a small file up to a larger allocation unit. + let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size(); + assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used); } #[test] diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 3f11400f34..83bc73093d 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -4985,6 +4985,25 @@ impl Volume { self.current_dat_file_size() } + /// Disk space held by the local .dat file, or 0 when it is unknown. It + /// exceeds the file size when the file was preallocated with + /// `FALLOC_FL_KEEP_SIZE`. + pub fn dat_allocated_size(&self) -> u64 { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let io = self.io.read().unwrap(); + io.dat_file + .as_ref() + .and_then(|f| f.metadata().ok()) + .map_or(0, |m| m.blocks() * 512) + } + #[cfg(not(unix))] + { + 0 + } + } + /// Get the modification time of the .dat file as Unix seconds. pub fn dat_file_mod_time(&self) -> u64 { let io = self.io.read().unwrap(); diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 5f468eb53a..ce76412a76 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -704,6 +704,8 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) continue } datSize, idxSize, _ := vol.FileStat() + // A preallocated volume already holds its space on disk. + datSize = max(datSize, vol.datAllocatedSize()) unUsedSpaceVolume := int64(volumeSizeLimit) - int64(datSize+idxSize) glog.V(4).Infof("Volume stats for %d: volumeSizeLimit=%d, datSize=%d idxSize=%d unused=%d", vol.Id, volumeSizeLimit, datSize, idxSize, unUsedSpaceVolume) if unUsedSpaceVolume >= 0 { diff --git a/weed/storage/disk_location_linux_test.go b/weed/storage/disk_location_linux_test.go new file mode 100644 index 0000000000..7815142c40 --- /dev/null +++ b/weed/storage/disk_location_linux_test.go @@ -0,0 +1,53 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +func TestUnUsedSpaceCountsPreallocatedVolumesAsFull(t *testing.T) { + const volumeSizeLimit = 1 << 20 + dir := t.TempDir() + + preallocated, err := backend.CreateVolumeFile(filepath.Join(dir, "1.dat"), volumeSizeLimit, 0) + if err != nil { + t.Fatal(err) + } + defer preallocated.Close() + fi, err := preallocated.(*backend.DiskFile).File.Stat() + if err != nil { + t.Fatal(err) + } + if allocatedSize(fi) < volumeSizeLimit { + t.Skipf("filesystem at %s does not support fallocate", dir) + } + + plainFile, err := os.Create(filepath.Join(dir, "2.dat")) + if err != nil { + t.Fatal(err) + } + if _, err := plainFile.Write(make([]byte, 4096)); err != nil { + t.Fatal(err) + } + plain := backend.NewDiskFile(plainFile) + defer plain.Close() + fi, err = plainFile.Stat() + if err != nil { + t.Fatal(err) + } + // The filesystem may round a small file up to a larger allocation unit. + plainSize := max(uint64(fi.Size()), allocatedSize(fi)) + + diskLocation := DiskLocation{Directory: dir, IdxDirectory: dir} + diskLocation.volumes = make(map[needle.VolumeId]*Volume) + diskLocation.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: preallocated, nm: &mockNeedleMapper{}}) + diskLocation.SetVolume(2, &Volume{Id: 2, dir: dir, dirIdx: dir, DataBackend: plain, nm: &mockNeedleMapper{}}) + + if got, want := diskLocation.UnUsedSpace(volumeSizeLimit), volumeSizeLimit-plainSize; got != want { + t.Errorf("unUsedSpace = %d, want %d", got, want) + } +} diff --git a/weed/storage/store.go b/weed/storage/store.go index d7fc750192..0186b0d269 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -1341,12 +1341,8 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) { if diskLocation.OriginalMaxVolumeCount == 0 { currentMaxVolumeCount := atomic.LoadInt32(&diskLocation.MaxVolumeCount) diskStatus := stats.NewDiskStatus(diskLocation.Directory) - var unusedSpace uint64 = 0 - unclaimedSpaces := int64(diskStatus.Free) - if !s.GetPreallocate() { - unusedSpace = diskLocation.UnUsedSpace(volumeSizeLimit) - unclaimedSpaces -= int64(unusedSpace) - } + unusedSpace := diskLocation.UnUsedSpace(volumeSizeLimit) + unclaimedSpaces := int64(diskStatus.Free) - int64(unusedSpace) volCount := diskLocation.VolumesLen() ecShardCount := diskLocation.EcShardCount() maxVolumeCount := int32(volCount) + int32((ecShardCount+erasure_coding.DataShardsCount-1)/erasure_coding.DataShardsCount) diff --git a/weed/storage/store_maxvolume_preallocate_test.go b/weed/storage/store_maxvolume_preallocate_test.go new file mode 100644 index 0000000000..64af0bdc2c --- /dev/null +++ b/weed/storage/store_maxvolume_preallocate_test.go @@ -0,0 +1,33 @@ +package storage + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +// A volume that was not preallocated keeps its remaining room reserved when +// the master turns preallocation on, so it still takes a slot of the free space. +func TestMaybeAdjustVolumeMaxReservesPlainVolumesWhilePreallocating(t *testing.T) { + dir := t.TempDir() + + // Room for 2.5 volumes: the plain volume claims one, leaving one more slot. + volumeSizeLimit := uint64(float64(stats.NewDiskStatus(dir).Free) / 2.5) + if volumeSizeLimit == 0 { + t.Skip("cannot determine free disk space") + } + + loc := &DiskLocation{Directory: dir, IdxDirectory: dir} + loc.volumes = make(map[needle.VolumeId]*Volume) + loc.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: &mockBackendStorageFile{}, nm: &mockNeedleMapper{}}) + s := &Store{Locations: []*DiskLocation{loc}} + s.SetVolumeSizeLimit(volumeSizeLimit) + s.SetPreallocate(true) + + s.MaybeAdjustVolumeMax() + + if loc.MaxVolumeCount != 2 { + t.Errorf("MaxVolumeCount=%d; want 2", loc.MaxVolumeCount) + } +} diff --git a/weed/storage/volume.go b/weed/storage/volume.go index ede48a494b..553435e5b8 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -204,6 +204,22 @@ func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) return // -1 causes integer overflow and the volume to become unwritable. } +// datAllocatedSize returns the disk space held by the .dat file, or 0 when it is unknown. +func (v *Volume) datAllocatedSize() uint64 { + v.dataFileAccessLock.RLock() + defer v.dataFileAccessLock.RUnlock() + + df, ok := v.DataBackend.(*backend.DiskFile) + if !ok { + return 0 + } + fi, err := df.File.Stat() + if err != nil { + return 0 + } + return allocatedSize(fi) +} + func (v *Volume) ContentSize() uint64 { v.dataFileAccessLock.RLock() defer v.dataFileAccessLock.RUnlock() diff --git a/weed/storage/volume_allocated_unix.go b/weed/storage/volume_allocated_unix.go new file mode 100644 index 0000000000..df267efd83 --- /dev/null +++ b/weed/storage/volume_allocated_unix.go @@ -0,0 +1,18 @@ +//go:build !windows + +package storage + +import ( + "os" + "syscall" +) + +// allocatedSize reports the bytes the filesystem holds for a file, which +// exceeds its size when the file was preallocated with FALLOC_FL_KEEP_SIZE. +func allocatedSize(fi os.FileInfo) uint64 { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + return uint64(st.Blocks) * 512 +} diff --git a/weed/storage/volume_allocated_windows.go b/weed/storage/volume_allocated_windows.go new file mode 100644 index 0000000000..6867c1ede9 --- /dev/null +++ b/weed/storage/volume_allocated_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package storage + +import "os" + +// allocatedSize is unknown on Windows, where volume files are never preallocated. +func allocatedSize(fi os.FileInfo) uint64 { + return 0 +}