diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index 3d633e72aa..5483274f9c 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -740,7 +740,11 @@ impl DiskLocation { if vol.is_read_only() { continue; } - let dat_size = vol.dat_file_size().unwrap_or(0); + // A preallocated volume already holds its space on disk. + let dat_size = vol + .dat_file_size() + .unwrap_or(0) + .max(vol.dat_allocated_size()); let idx_size = vol.idx_file_size(); let used = dat_size + idx_size; if volume_size_limit > used { diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 014705c9f7..e1e12fb608 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -943,11 +943,7 @@ impl Store { let current = loc.max_volume_count.load(Ordering::Relaxed); let (_, free) = super::disk_location::get_disk_stats(&loc.directory); - let unused_space = if self.get_preallocate() { - 0 - } else { - loc.unused_space(volume_size_limit) - }; + let unused_space = loc.unused_space(volume_size_limit); let unclaimed = (free as i64) - (unused_space as i64); let vol_count = loc.volumes_len() as i32; @@ -2479,7 +2475,7 @@ mod tests { } #[test] - fn test_maybe_adjust_volume_max_honors_preallocate_flag() { + fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() { let tmp = TempDir::new().unwrap(); let dir = tmp.path().to_str().unwrap(); @@ -2494,13 +2490,6 @@ mod tests { Vec::new(), ) .unwrap(); - // Use a large volume_size_limit so the unused-space difference between - // preallocate=true (0) and preallocate=false (~2 × limit) is big enough - // that integer-division rounding and disk-free fluctuations between the - // two get_disk_stats calls cannot make the quotients equal. - store - .volume_size_limit - .store(100 * 1024 * 1024, Ordering::Relaxed); store .add_volume( VolumeId(61), @@ -2511,39 +2500,67 @@ mod tests { }, ) .unwrap(); + + // Room for 2.5 volumes: the plain volume claims one, leaving one more slot. + let (_, free) = crate::storage::disk_location::get_disk_stats(dir); + let volume_size_limit = (free as f64 / 2.5) as u64; + if volume_size_limit == 0 { + return; // cannot determine free disk space + } store - .add_volume( - VolumeId(62), + .volume_size_limit + .store(volume_size_limit, Ordering::Relaxed); + store.locations[0].original_max_volume_count = 0; + store.set_preallocate(true); + + store.maybe_adjust_volume_max(); + assert_eq!( + store.locations[0].max_volume_count.load(Ordering::Relaxed), + 2 + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn test_unused_space_counts_preallocated_volumes_as_full() { + const LIMIT: u64 = 1 << 20; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + let mut store = Store::new(NeedleMapKind::InMemory); + store + .add_location( + dir, + dir, + 2, DiskType::HardDrive, - &VolumeSpec { - collection: "preallocate_case", - ..Default::default() - }, + MinFreeSpace::Percent(1.0), + Vec::new(), ) .unwrap(); - for vid in [VolumeId(61), VolumeId(62)] { - let dat_path = store.find_volume(vid).unwrap().1.dat_path(); - std::fs::OpenOptions::new() - .write(true) - .open(dat_path) - .unwrap() - .set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64) + for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] { + store + .add_volume( + vid, + DiskType::HardDrive, + &VolumeSpec { + collection: "preallocate_case", + preallocate, + ..Default::default() + }, + ) .unwrap(); } - store.locations[0].original_max_volume_count = 0; - store.locations[0] - .max_volume_count - .store(0, Ordering::Relaxed); + let preallocated = store.find_volume(VolumeId(71)).unwrap().1; + if preallocated.dat_allocated_size() < LIMIT { + return; // the filesystem does not support fallocate + } - store.set_preallocate(false); - assert!(store.maybe_adjust_volume_max()); - let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed); - - store.set_preallocate(true); - assert!(store.maybe_adjust_volume_max()); - let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed); - - assert!(with_preallocate > without_preallocate); + let plain = store.find_volume(VolumeId(72)).unwrap().1; + let plain_dat = plain.dat_file_size().unwrap(); + // The filesystem may round a small file up to a larger allocation unit. + let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size(); + assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used); } #[test] diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 3f11400f34..83bc73093d 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -4985,6 +4985,25 @@ impl Volume { self.current_dat_file_size() } + /// Disk space held by the local .dat file, or 0 when it is unknown. It + /// exceeds the file size when the file was preallocated with + /// `FALLOC_FL_KEEP_SIZE`. + pub fn dat_allocated_size(&self) -> u64 { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let io = self.io.read().unwrap(); + io.dat_file + .as_ref() + .and_then(|f| f.metadata().ok()) + .map_or(0, |m| m.blocks() * 512) + } + #[cfg(not(unix))] + { + 0 + } + } + /// Get the modification time of the .dat file as Unix seconds. pub fn dat_file_mod_time(&self) -> u64 { let io = self.io.read().unwrap(); diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 5f468eb53a..ce76412a76 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -704,6 +704,8 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) continue } datSize, idxSize, _ := vol.FileStat() + // A preallocated volume already holds its space on disk. + datSize = max(datSize, vol.datAllocatedSize()) unUsedSpaceVolume := int64(volumeSizeLimit) - int64(datSize+idxSize) glog.V(4).Infof("Volume stats for %d: volumeSizeLimit=%d, datSize=%d idxSize=%d unused=%d", vol.Id, volumeSizeLimit, datSize, idxSize, unUsedSpaceVolume) if unUsedSpaceVolume >= 0 { diff --git a/weed/storage/disk_location_linux_test.go b/weed/storage/disk_location_linux_test.go new file mode 100644 index 0000000000..7815142c40 --- /dev/null +++ b/weed/storage/disk_location_linux_test.go @@ -0,0 +1,53 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +func TestUnUsedSpaceCountsPreallocatedVolumesAsFull(t *testing.T) { + const volumeSizeLimit = 1 << 20 + dir := t.TempDir() + + preallocated, err := backend.CreateVolumeFile(filepath.Join(dir, "1.dat"), volumeSizeLimit, 0) + if err != nil { + t.Fatal(err) + } + defer preallocated.Close() + fi, err := preallocated.(*backend.DiskFile).File.Stat() + if err != nil { + t.Fatal(err) + } + if allocatedSize(fi) < volumeSizeLimit { + t.Skipf("filesystem at %s does not support fallocate", dir) + } + + plainFile, err := os.Create(filepath.Join(dir, "2.dat")) + if err != nil { + t.Fatal(err) + } + if _, err := plainFile.Write(make([]byte, 4096)); err != nil { + t.Fatal(err) + } + plain := backend.NewDiskFile(plainFile) + defer plain.Close() + fi, err = plainFile.Stat() + if err != nil { + t.Fatal(err) + } + // The filesystem may round a small file up to a larger allocation unit. + plainSize := max(uint64(fi.Size()), allocatedSize(fi)) + + diskLocation := DiskLocation{Directory: dir, IdxDirectory: dir} + diskLocation.volumes = make(map[needle.VolumeId]*Volume) + diskLocation.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: preallocated, nm: &mockNeedleMapper{}}) + diskLocation.SetVolume(2, &Volume{Id: 2, dir: dir, dirIdx: dir, DataBackend: plain, nm: &mockNeedleMapper{}}) + + if got, want := diskLocation.UnUsedSpace(volumeSizeLimit), volumeSizeLimit-plainSize; got != want { + t.Errorf("unUsedSpace = %d, want %d", got, want) + } +} diff --git a/weed/storage/store.go b/weed/storage/store.go index d7fc750192..0186b0d269 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -1341,12 +1341,8 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) { if diskLocation.OriginalMaxVolumeCount == 0 { currentMaxVolumeCount := atomic.LoadInt32(&diskLocation.MaxVolumeCount) diskStatus := stats.NewDiskStatus(diskLocation.Directory) - var unusedSpace uint64 = 0 - unclaimedSpaces := int64(diskStatus.Free) - if !s.GetPreallocate() { - unusedSpace = diskLocation.UnUsedSpace(volumeSizeLimit) - unclaimedSpaces -= int64(unusedSpace) - } + unusedSpace := diskLocation.UnUsedSpace(volumeSizeLimit) + unclaimedSpaces := int64(diskStatus.Free) - int64(unusedSpace) volCount := diskLocation.VolumesLen() ecShardCount := diskLocation.EcShardCount() maxVolumeCount := int32(volCount) + int32((ecShardCount+erasure_coding.DataShardsCount-1)/erasure_coding.DataShardsCount) diff --git a/weed/storage/store_maxvolume_preallocate_test.go b/weed/storage/store_maxvolume_preallocate_test.go new file mode 100644 index 0000000000..64af0bdc2c --- /dev/null +++ b/weed/storage/store_maxvolume_preallocate_test.go @@ -0,0 +1,33 @@ +package storage + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +// A volume that was not preallocated keeps its remaining room reserved when +// the master turns preallocation on, so it still takes a slot of the free space. +func TestMaybeAdjustVolumeMaxReservesPlainVolumesWhilePreallocating(t *testing.T) { + dir := t.TempDir() + + // Room for 2.5 volumes: the plain volume claims one, leaving one more slot. + volumeSizeLimit := uint64(float64(stats.NewDiskStatus(dir).Free) / 2.5) + if volumeSizeLimit == 0 { + t.Skip("cannot determine free disk space") + } + + loc := &DiskLocation{Directory: dir, IdxDirectory: dir} + loc.volumes = make(map[needle.VolumeId]*Volume) + loc.SetVolume(1, &Volume{Id: 1, dir: dir, dirIdx: dir, DataBackend: &mockBackendStorageFile{}, nm: &mockNeedleMapper{}}) + s := &Store{Locations: []*DiskLocation{loc}} + s.SetVolumeSizeLimit(volumeSizeLimit) + s.SetPreallocate(true) + + s.MaybeAdjustVolumeMax() + + if loc.MaxVolumeCount != 2 { + t.Errorf("MaxVolumeCount=%d; want 2", loc.MaxVolumeCount) + } +} diff --git a/weed/storage/volume.go b/weed/storage/volume.go index ede48a494b..553435e5b8 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -204,6 +204,22 @@ func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) return // -1 causes integer overflow and the volume to become unwritable. } +// datAllocatedSize returns the disk space held by the .dat file, or 0 when it is unknown. +func (v *Volume) datAllocatedSize() uint64 { + v.dataFileAccessLock.RLock() + defer v.dataFileAccessLock.RUnlock() + + df, ok := v.DataBackend.(*backend.DiskFile) + if !ok { + return 0 + } + fi, err := df.File.Stat() + if err != nil { + return 0 + } + return allocatedSize(fi) +} + func (v *Volume) ContentSize() uint64 { v.dataFileAccessLock.RLock() defer v.dataFileAccessLock.RUnlock() diff --git a/weed/storage/volume_allocated_unix.go b/weed/storage/volume_allocated_unix.go new file mode 100644 index 0000000000..df267efd83 --- /dev/null +++ b/weed/storage/volume_allocated_unix.go @@ -0,0 +1,18 @@ +//go:build !windows + +package storage + +import ( + "os" + "syscall" +) + +// allocatedSize reports the bytes the filesystem holds for a file, which +// exceeds its size when the file was preallocated with FALLOC_FL_KEEP_SIZE. +func allocatedSize(fi os.FileInfo) uint64 { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + return uint64(st.Blocks) * 512 +} diff --git a/weed/storage/volume_allocated_windows.go b/weed/storage/volume_allocated_windows.go new file mode 100644 index 0000000000..6867c1ede9 --- /dev/null +++ b/weed/storage/volume_allocated_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package storage + +import "os" + +// allocatedSize is unknown on Windows, where volume files are never preallocated. +func allocatedSize(fi os.FileInfo) uint64 { + return 0 +}