fix(storage): count preallocated space when auto-sizing max volumes (#11919)

The max volume count subtracted each volume's remaining room from the
free disk space, except when the master had preallocation turned on,
in which case it subtracted nothing. That is wrong for a disk that
mixes preallocated and plain volumes, and on platforms where
preallocation is not supported.

Use the larger of the .dat file's size and its allocated blocks, in the
Go and the Rust volume server, so a preallocated volume counts as full
and a plain one keeps its remaining room reserved.

Fixes #5778

Co-authored-by: Dan Foster <4570306+danfoster@users.noreply.github.com>

seaweed-volume/src/storage/disk_location.rs

@@ -740,7 +740,11 @@ impl DiskLocation {
             if vol.is_read_only() {
                 continue;
             }
-            let dat_size = vol.dat_file_size().unwrap_or(0);
+            // A preallocated volume already holds its space on disk.
+            let dat_size = vol
+                .dat_file_size()
+                .unwrap_or(0)
+                .max(vol.dat_allocated_size());
             let idx_size = vol.idx_file_size();
             let used = dat_size + idx_size;
             if volume_size_limit > used {

seaweed-volume/src/storage/store.rs

@@ -943,11 +943,7 @@ impl Store {
                 let current = loc.max_volume_count.load(Ordering::Relaxed);
                 let (_, free) = super::disk_location::get_disk_stats(&loc.directory);
 
-                let unused_space = if self.get_preallocate() {
-                    0
-                } else {
-                    loc.unused_space(volume_size_limit)
-                };
+                let unused_space = loc.unused_space(volume_size_limit);
                 let unclaimed = (free as i64) - (unused_space as i64);
 
                 let vol_count = loc.volumes_len() as i32;
@@ -2479,7 +2475,7 @@ mod tests {
     }
 
     #[test]
-    fn test_maybe_adjust_volume_max_honors_preallocate_flag() {
+    fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() {
         let tmp = TempDir::new().unwrap();
         let dir = tmp.path().to_str().unwrap();
 
@@ -2494,13 +2490,6 @@ mod tests {
                 Vec::new(),
             )
             .unwrap();
-        // Use a large volume_size_limit so the unused-space difference between
-        // preallocate=true (0) and preallocate=false (~2 × limit) is big enough
-        // that integer-division rounding and disk-free fluctuations between the
-        // two get_disk_stats calls cannot make the quotients equal.
-        store
-            .volume_size_limit
-            .store(100 * 1024 * 1024, Ordering::Relaxed);
         store
             .add_volume(
                 VolumeId(61),
@@ -2511,39 +2500,67 @@ mod tests {
                 },
             )
             .unwrap();
+
+        // Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
+        let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
+        let volume_size_limit = (free as f64 / 2.5) as u64;
+        if volume_size_limit == 0 {
+            return; // cannot determine free disk space
+        }
         store
-            .add_volume(
-                VolumeId(62),
+            .volume_size_limit
+            .store(volume_size_limit, Ordering::Relaxed);
+        store.locations[0].original_max_volume_count = 0;
+        store.set_preallocate(true);
+
+        store.maybe_adjust_volume_max();
+        assert_eq!(
+            store.locations[0].max_volume_count.load(Ordering::Relaxed),
+            2
+        );
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn test_unused_space_counts_preallocated_volumes_as_full() {
+        const LIMIT: u64 = 1 << 20;
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        store
+            .add_location(
+                dir,
+                dir,
+                2,
                 DiskType::HardDrive,
-                &VolumeSpec {
-                    collection: "preallocate_case",
-                    ..Default::default()
-                },
+                MinFreeSpace::Percent(1.0),
+                Vec::new(),
             )
             .unwrap();
-        for vid in [VolumeId(61), VolumeId(62)] {
-            let dat_path = store.find_volume(vid).unwrap().1.dat_path();
-            std::fs::OpenOptions::new()
-                .write(true)
-                .open(dat_path)
-                .unwrap()
-                .set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64)
+        for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] {
+            store
+                .add_volume(
+                    vid,
+                    DiskType::HardDrive,
+                    &VolumeSpec {
+                        collection: "preallocate_case",
+                        preallocate,
+                        ..Default::default()
+                    },
+                )
                 .unwrap();
         }
-        store.locations[0].original_max_volume_count = 0;
-        store.locations[0]
-            .max_volume_count
-            .store(0, Ordering::Relaxed);
+        let preallocated = store.find_volume(VolumeId(71)).unwrap().1;
+        if preallocated.dat_allocated_size() < LIMIT {
+            return; // the filesystem does not support fallocate
+        }
 
-        store.set_preallocate(false);
-        assert!(store.maybe_adjust_volume_max());
-        let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
-
-        store.set_preallocate(true);
-        assert!(store.maybe_adjust_volume_max());
-        let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
-
-        assert!(with_preallocate > without_preallocate);
+        let plain = store.find_volume(VolumeId(72)).unwrap().1;
+        let plain_dat = plain.dat_file_size().unwrap();
+        // The filesystem may round a small file up to a larger allocation unit.
+        let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size();
+        assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used);
     }
 
     #[test]

seaweed-volume/src/storage/volume.rs

@@ -4985,6 +4985,25 @@ impl Volume {
         self.current_dat_file_size()
     }
 
+    /// Disk space held by the local .dat file, or 0 when it is unknown. It
+    /// exceeds the file size when the file was preallocated with
+    /// `FALLOC_FL_KEEP_SIZE`.
+    pub fn dat_allocated_size(&self) -> u64 {
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::MetadataExt;
+            let io = self.io.read().unwrap();
+            io.dat_file
+                .as_ref()
+                .and_then(|f| f.metadata().ok())
+                .map_or(0, |m| m.blocks() * 512)
+        }
+        #[cfg(not(unix))]
+        {
+            0
+        }
+    }
+
     /// Get the modification time of the .dat file as Unix seconds.
     pub fn dat_file_mod_time(&self) -> u64 {
         let io = self.io.read().unwrap();

weed/storage/disk_location.go

@@ -704,6 +704,8 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64)
 			continue
 		}
 		datSize, idxSize, _ := vol.FileStat()
+		// A preallocated volume already holds its space on disk.
+		datSize = max(datSize, vol.datAllocatedSize())
 		unUsedSpaceVolume := int64(volumeSizeLimit) - int64(datSize+idxSize)
 		glog.V(4).Infof("Volume stats for %d: volumeSizeLimit=%d, datSize=%d idxSize=%d unused=%d", vol.Id, volumeSizeLimit, datSize, idxSize, unUsedSpaceVolume)
 		if unUsedSpaceVolume >= 0 {

weed/storage/disk_location_linux_test.go

@@ -0,0 +1,53 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestUnUsedSpaceCountsPreallocatedVolumesAsFull(t *testing.T) {
+	const volumeSizeLimit = 1 << 20
+	dir := t.TempDir()
+
+	preallocated, err := backend.CreateVolumeFile(filepath.Join(dir, "1.dat"), volumeSizeLimit, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer preallocated.Close()
+	fi, err := preallocated.(*backend.DiskFile).File.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if allocatedSize(fi) < volumeSizeLimit {
+		t.Skipf("filesystem at %s does not support fallocate", dir)
+	}
+
+	plainFile, err := os.Create(filepath.Join(dir, "2.dat"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := plainFile.Write(make([]byte, 4096)); err != nil {
+		t.Fatal(err)
+	}
+	plain := backend.NewDiskFile(plainFile)
+	defer plain.Close()
+	fi, err = plainFile.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The filesystem may round a small file up to a larger allocation unit.
+	plainSize := max(uint64(fi.Size()), allocatedSize(fi))
+
+	diskLocation := DiskLocation{Directory: dir, IdxDirectory: dir}
+	diskLocation.volumes = make(map[needle.VolumeId]*Volume)
+	diskLocation.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: preallocated, nm: &mockNeedleMapper{}})
+	diskLocation.SetVolume(2, &Volume{Id: 2, dir: dir, dirIdx: dir, DataBackend: plain, nm: &mockNeedleMapper{}})
+
+	if got, want := diskLocation.UnUsedSpace(volumeSizeLimit), volumeSizeLimit-plainSize; got != want {
+		t.Errorf("unUsedSpace = %d, want %d", got, want)
+	}
+}

weed/storage/store.go

@@ -1341,12 +1341,8 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
 		if diskLocation.OriginalMaxVolumeCount == 0 {
 			currentMaxVolumeCount := atomic.LoadInt32(&diskLocation.MaxVolumeCount)
 			diskStatus := stats.NewDiskStatus(diskLocation.Directory)
-			var unusedSpace uint64 = 0
-			unclaimedSpaces := int64(diskStatus.Free)
-			if !s.GetPreallocate() {
-				unusedSpace = diskLocation.UnUsedSpace(volumeSizeLimit)
-				unclaimedSpaces -= int64(unusedSpace)
-			}
+			unusedSpace := diskLocation.UnUsedSpace(volumeSizeLimit)
+			unclaimedSpaces := int64(diskStatus.Free) - int64(unusedSpace)
 			volCount := diskLocation.VolumesLen()
 			ecShardCount := diskLocation.EcShardCount()
 			maxVolumeCount := int32(volCount) + int32((ecShardCount+erasure_coding.DataShardsCount-1)/erasure_coding.DataShardsCount)

weed/storage/store_maxvolume_preallocate_test.go

@@ -0,0 +1,33 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// A volume that was not preallocated keeps its remaining room reserved when
+// the master turns preallocation on, so it still takes a slot of the free space.
+func TestMaybeAdjustVolumeMaxReservesPlainVolumesWhilePreallocating(t *testing.T) {
+	dir := t.TempDir()
+
+	// Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
+	volumeSizeLimit := uint64(float64(stats.NewDiskStatus(dir).Free) / 2.5)
+	if volumeSizeLimit == 0 {
+		t.Skip("cannot determine free disk space")
+	}
+
+	loc := &DiskLocation{Directory: dir, IdxDirectory: dir}
+	loc.volumes = make(map[needle.VolumeId]*Volume)
+	loc.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: &mockBackendStorageFile{}, nm: &mockNeedleMapper{}})
+	s := &Store{Locations: []*DiskLocation{loc}}
+	s.SetVolumeSizeLimit(volumeSizeLimit)
+	s.SetPreallocate(true)
+
+	s.MaybeAdjustVolumeMax()
+
+	if loc.MaxVolumeCount != 2 {
+		t.Errorf("MaxVolumeCount=%d; want 2", loc.MaxVolumeCount)
+	}
+}

weed/storage/volume.go

@@ -204,6 +204,22 @@ func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time)
 	return // -1 causes integer overflow and the volume to become unwritable.
 }
 
+// datAllocatedSize returns the disk space held by the .dat file, or 0 when it is unknown.
+func (v *Volume) datAllocatedSize() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+
+	df, ok := v.DataBackend.(*backend.DiskFile)
+	if !ok {
+		return 0
+	}
+	fi, err := df.File.Stat()
+	if err != nil {
+		return 0
+	}
+	return allocatedSize(fi)
+}
+
 func (v *Volume) ContentSize() uint64 {
 	v.dataFileAccessLock.RLock()
 	defer v.dataFileAccessLock.RUnlock()

weed/storage/volume_allocated_unix.go

@@ -0,0 +1,18 @@
+//go:build !windows
+
+package storage
+
+import (
+	"os"
+	"syscall"
+)
+
+// allocatedSize reports the bytes the filesystem holds for a file, which
+// exceeds its size when the file was preallocated with FALLOC_FL_KEEP_SIZE.
+func allocatedSize(fi os.FileInfo) uint64 {
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0
+	}
+	return uint64(st.Blocks) * 512
+}

weed/storage/volume_allocated_windows.go

@@ -0,0 +1,10 @@
+//go:build windows
+
+package storage
+
+import "os"
+
+// allocatedSize is unknown on Windows, where volume files are never preallocated.
+func allocatedSize(fi os.FileInfo) uint64 {
+	return 0
+}