perf(cache): drop OS page cache after disk cache reads (#9098)

* perf(cache): drop OS page cache after disk cache reads

After reading from the on-disk chunk cache, advise the kernel via
FADV_DONTNEED to release the corresponding page cache pages. This
prevents double-caching the same data in both user-space and kernel
page caches, freeing RAM for other uses on systems with large disk
caches.

* fix(cache): guard dropReadCache against zero length and invalid fd

A zero-length fadvise is interpreted as "to end of file" on Linux,
which would inadvertently drop the page cache for the entire remainder
of the cache volume. Also check fd >= 0 to avoid unnecessary syscalls
when the backend file is closed.

* perf(cache): only apply FADV_DONTNEED for reads >= 1 MiB

For small needle reads the syscall overhead outweighs the memory
savings, and the kernel page cache is more beneficial for warm data.
Restrict fadvise to reads of at least 1 MiB where the freed page
cache is meaningful.

weed/storage/backend/disk_file.go

@@ -115,6 +115,13 @@ func (df *DiskFile) Name() string {
 	return df.fullFilePath
 }
 
+func (df *DiskFile) Fd() uintptr {
+	if df.File == nil {
+		return ^uintptr(0)
+	}
+	return df.File.Fd()
+}
+
 func (df *DiskFile) Sync() error {
 	if df.File == nil {
 		return os.ErrClosed

weed/util/chunk_cache/chunk_cache_on_disk.go

@@ -114,6 +114,33 @@ func (v *ChunkCacheVolume) Reset() (*ChunkCacheVolume, error) {
 	return LoadOrCreateChunkCacheVolume(v.fileName, v.sizeLimit)
 }
 
+// minFadviseSize is the minimum read size (in bytes) before we call fadvise
+// DONTNEED. For small reads the syscall overhead outweighs the benefit, and
+// the kernel's page cache may serve the data again sooner than we think.
+const minFadviseSize = 1 << 20 // 1 MiB
+
+// dropReadCache advises the kernel to drop page cache for the byte range
+// just read. This is best-effort; failures are logged at V(4).
+// Only applied for reads >= minFadviseSize to avoid syscall overhead on
+// small needle reads where the kernel page cache is more beneficial.
+func (v *ChunkCacheVolume) dropReadCache(offset int64, length int64) {
+	if length < minFadviseSize {
+		return
+	}
+	type fdProvider interface {
+		Fd() uintptr
+	}
+	if fp, ok := v.DataBackend.(fdProvider); ok {
+		fd := int(fp.Fd())
+		if fd < 0 {
+			return
+		}
+		if err := util.DropOSPageCache(fd, offset, length); err != nil {
+			glog.V(4).Infof("fadvise DONTNEED %s offset %d len %d: %v", v.fileName, offset, length, err)
+		}
+	}
+}
+
 func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) {
 
 	nv, ok := v.nm.Get(key)
@@ -121,10 +148,11 @@ func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) {
 		return nil, storage.ErrorNotFound
 	}
 	data := make([]byte, nv.Size)
-	if readSize, readErr := v.DataBackend.ReadAt(data, nv.Offset.ToActualOffset()); readErr != nil {
+	readOffset := nv.Offset.ToActualOffset()
+	if readSize, readErr := v.DataBackend.ReadAt(data, readOffset); readErr != nil {
 		if readSize != int(nv.Size) {
 			return nil, fmt.Errorf("read %s.dat [%d,%d): %v",
-				v.fileName, nv.Offset.ToActualOffset(), nv.Offset.ToActualOffset()+int64(nv.Size), readErr)
+				v.fileName, readOffset, readOffset+int64(nv.Size), readErr)
 		}
 	} else {
 		if readSize != int(nv.Size) {
@@ -132,6 +160,7 @@ func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) {
 		}
 	}
 
+	v.dropReadCache(readOffset, int64(nv.Size))
 	return data, nil
 }
 
@@ -146,12 +175,13 @@ func (v *ChunkCacheVolume) getNeedleSlice(key types.NeedleId, offset, length uin
 		return nil, ErrorOutOfBounds
 	}
 	data := make([]byte, wanted)
+	readOffset := nv.Offset.ToActualOffset() + int64(offset)
 	var readSize int
 	var readErr error
-	if readSize, readErr = v.DataBackend.ReadAt(data, nv.Offset.ToActualOffset()+int64(offset)); readErr != nil {
+	if readSize, readErr = v.DataBackend.ReadAt(data, readOffset); readErr != nil {
 		if readSize != wanted {
 			return nil, fmt.Errorf("read %s.dat [%d,%d): %v",
-				v.fileName, nv.Offset.ToActualOffset()+int64(offset), int(nv.Offset.ToActualOffset())+int(offset)+wanted, readErr)
+				v.fileName, readOffset, int64(readOffset)+int64(wanted), readErr)
 		}
 	} else {
 		if readSize != wanted {
@@ -161,6 +191,9 @@ func (v *ChunkCacheVolume) getNeedleSlice(key types.NeedleId, offset, length uin
 	if readErr != nil && readSize == wanted {
 		readErr = nil
 	}
+	if readSize > 0 {
+		v.dropReadCache(readOffset, int64(readSize))
+	}
 	return data, readErr
 }
 
@@ -174,10 +207,11 @@ func (v *ChunkCacheVolume) readNeedleSliceAt(data []byte, key types.NeedleId, of
 		// should never happen, but better than panicking
 		return 0, ErrorOutOfBounds
 	}
-	if n, err = v.DataBackend.ReadAt(data, nv.Offset.ToActualOffset()+int64(offset)); err != nil {
+	readOffset := nv.Offset.ToActualOffset() + int64(offset)
+	if n, err = v.DataBackend.ReadAt(data, readOffset); err != nil {
 		if n != wanted {
 			return n, fmt.Errorf("read %s.dat [%d,%d): %v",
-				v.fileName, nv.Offset.ToActualOffset()+int64(offset), int(nv.Offset.ToActualOffset())+int(offset)+wanted, err)
+				v.fileName, readOffset, int64(readOffset)+int64(wanted), err)
 		}
 	} else {
 		if n != wanted {
@@ -187,6 +221,9 @@ func (v *ChunkCacheVolume) readNeedleSliceAt(data []byte, key types.NeedleId, of
 	if err != nil && n == wanted {
 		err = nil
 	}
+	if n > 0 {
+		v.dropReadCache(readOffset, int64(n))
+	}
 	return n, err
 }
 

weed/util/fadvise_linux.go

@@ -0,0 +1,13 @@
+//go:build linux
+
+package util
+
+import "golang.org/x/sys/unix"
+
+// DropOSPageCache advises the kernel that the given byte range is no longer
+// needed in the page cache. This is useful after reading from a user-space
+// cache (e.g., on-disk chunk cache) to prevent the kernel from double-caching
+// the same data.
+func DropOSPageCache(fd int, offset int64, length int64) error {
+	return unix.Fadvise(fd, offset, length, unix.FADV_DONTNEED)
+}

weed/util/fadvise_linux_test.go

@@ -0,0 +1,37 @@
+//go:build linux
+
+package util
+
+import (
+	"os"
+	"testing"
+)
+
+func TestDropOSPageCache(t *testing.T) {
+	f, err := os.CreateTemp("", "fadvise_test")
+	if err != nil {
+		t.Fatalf("failed to create temp file: %v", err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+
+	data := make([]byte, 4096)
+	for i := range data {
+		data[i] = byte(i % 256)
+	}
+	if _, err := f.Write(data); err != nil {
+		t.Fatalf("failed to write test data: %v", err)
+	}
+
+	// Read the data back to populate page cache
+	buf := make([]byte, 4096)
+	if _, err := f.ReadAt(buf, 0); err != nil {
+		t.Fatalf("failed to read test data: %v", err)
+	}
+
+	// Call DropOSPageCache and verify no error
+	fd := int(f.Fd())
+	if err := DropOSPageCache(fd, 0, int64(len(data))); err != nil {
+		t.Errorf("DropOSPageCache returned error: %v", err)
+	}
+}

weed/util/fadvise_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package util
+
+// DropOSPageCache is a no-op on non-Linux platforms.
+func DropOSPageCache(fd int, offset int64, length int64) error {
+	return nil
+}