seaweedfs/weed/storage/volume.go

package storage

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
	"github.com/seaweedfs/seaweedfs/weed/util"

	"github.com/seaweedfs/seaweedfs/weed/glog"
)

type Volume struct {
	Id                 needle.VolumeId
	dir                string
	dirIdx             string
	Collection         string
	DataBackend        backend.BackendStorageFile
	nm                 NeedleMapper
	tmpNm              TempNeedleMapper
	needleMapKind      NeedleMapKind
	noWriteOrDelete    bool // if readonly, either noWriteOrDelete or noWriteCanDelete
	noWriteCanDelete   bool // if readonly, either noWriteOrDelete or noWriteCanDelete
	ioUnavailable      bool
	ioUnavailableError string
	noWriteLock        sync.RWMutex
	hasRemoteFile      atomic.Bool // if the volume is tiered: data lives in a remote backend
	MemoryMapMaxSizeMb uint32

	super_block.SuperBlock

	dataFileAccessLock sync.RWMutex
	// Serializes a data-file mutation with the sync that makes it durable:
	// write, delete, lifecycle and compaction paths hold it for their whole
	// critical section, while the fsync itself runs with dataFileAccessLock
	// released so readers of this volume do not wait on it.
	dataFileSyncLock     sync.Mutex
	superBlockAccessLock sync.Mutex

	// The batch worker exists only once the volume takes a durable write. Most
	// never do -- read-only, remote-tiered, or written without fsync -- and a
	// parked worker costs its goroutine stack plus a 128-slot channel, which a
	// server holding millions of volumes cannot pay for all of them.
	asyncWorkerLock   sync.Mutex
	asyncRequestsChan chan *needle.AsyncRequest
	asyncWorkerClosed bool

	lastModifiedTsSeconds    uint64         // unix time in seconds
	lastAppendAtNs           uint64         // unix time in nanoseconds
	lastWriteAppendAtNs      uint64         // AppendAtNs of the newest write; tombstones don't move it
	lastWriteNeedleKey       types.NeedleId // the write behind the watermark
	lastWriteDeleted         bool           // that write was deleted, so recovery has to rescan
	keepLastModifiedTsOnLoad bool

	lastCompactIndexOffset uint64
	lastCompactRevision    uint16
	ldbTimeout             int64

	isCompactionInProgress atomic.Bool
	lastDiskCheckNs        atomic.Int64 // unix time in nanoseconds for phantom volume detection

	volumeInfoRWLock sync.RWMutex
	volumeInfo       *volume_server_pb.VolumeInfo
	location         *DiskLocation
	diskId           uint32 // ID of this volume's disk in Store.Locations array

	IoErrorTracker
}

func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, ver needle.Version, memoryMapMaxSizeMb uint32, ldbTimeout int64) (v *Volume, e error) {
	// if replicaPlacement is nil, the superblock will be loaded from disk
	v = &Volume{dir: dirname, dirIdx: dirIdx, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb}
	v.SuperBlock = super_block.SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
	v.needleMapKind = needleMapKind
	v.ldbTimeout = ldbTimeout
	e = v.load(true, true, needleMapKind, preallocate, ver)
	return
}

func (v *Volume) String() string {
	v.noWriteLock.RLock()
	defer v.noWriteLock.RUnlock()
	return fmt.Sprintf("Id:%v dir:%s dirIdx:%s Collection:%s dataFile:%v nm:%v noWrite:%v canDelete:%v", v.Id, v.dir, v.dirIdx, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
}

func VolumeFileName(dir string, collection string, id int) (fileName string) {
	idString := strconv.Itoa(id)
	if collection == "" {
		fileName = path.Join(dir, idString)
	} else {
		fileName = path.Join(dir, collection+"_"+idString)
	}
	return
}

func (v *Volume) DataFileName() (fileName string) {
	return VolumeFileName(v.dir, v.Collection, int(v.Id))
}

func (v *Volume) IndexFileName() (fileName string) {
	return VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
}

func (v *Volume) FileName(ext string) (fileName string) {
	switch ext {
	case ".idx", ".cpx", ".ldb", ".cpldb", ".rdb":
		return VolumeFileName(v.dirIdx, v.Collection, int(v.Id)) + ext
	}
	// .dat, .cpd, .vif
	return VolumeFileName(v.dir, v.Collection, int(v.Id)) + ext
}

// RelocateIndexTo moves the volume's index to newIdxDir and reopens the volume
// against it in place, without unmounting. It takes the data-file write lock —
// so a concurrent read blocks briefly instead of failing — closes the needle
// map and data backend, moves the .idx (and the derived .sdx best-effort), then
// retargets dirIdx and reloads, mirroring CommitCompact's close-swap-load. A
// decode co-locates the rebuilt index with the data so the on-demand mount can
// find the volume; this returns it to the -dir.idx tier once the EC shards are
// gone. A no-op when the index already lives in newIdxDir. A derived .ldb is
// not moved: the reload rebuilds it in newIdxDir from the .idx.
func (v *Volume) RelocateIndexTo(newIdxDir string) error {
	v.dataFileSyncLock.Lock()
	defer v.dataFileSyncLock.Unlock()
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()

	if v.dirIdx == newIdxDir {
		return nil
	}
	oldBase := VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
	if _, err := os.Stat(oldBase + ".idx"); err != nil {
		return nil // nothing co-located to move
	}
	newBase := VolumeFileName(newIdxDir, v.Collection, int(v.Id))

	if v.nm != nil {
		_ = v.nm.Sync()
		v.nm.Close()
		v.nm = nil
	}
	if v.DataBackend != nil {
		_ = v.DataBackend.Sync()
		_ = v.DataBackend.Close()
		v.DataBackend = nil
	}

	if err := RenameOrCopyFile(oldBase+".idx", newBase+".idx"); err != nil {
		// Reopen against the old dir so the volume is not left down; surface a
		// failed reopen since it leaves the volume unusable until the next load.
		if reopenErr := v.load(true, false, v.needleMapKind, 0, v.Version()); reopenErr != nil {
			glog.Errorf("relocate volume %d: reopen after failed .idx move: %v", v.Id, reopenErr)
		}
		return fmt.Errorf("relocate index for volume %d: move .idx: %w", v.Id, err)
	}
	// The .sdx is a derived sorted index; move it when present, but a failure is
	// not fatal — drop the stale copy so the reload rebuilds it in the new dir.
	if _, err := os.Stat(oldBase + ".sdx"); err == nil {
		if err := RenameOrCopyFile(oldBase+".sdx", newBase+".sdx"); err != nil {
			glog.Warningf("relocate volume %d: move .sdx: %v (will rebuild)", v.Id, err)
			_ = os.Remove(oldBase + ".sdx")
		}
	}
	v.dirIdx = newIdxDir
	return v.load(true, false, v.needleMapKind, 0, v.Version())
}

func (v *Volume) Version() needle.Version {
	v.superBlockAccessLock.Lock()
	defer v.superBlockAccessLock.Unlock()
	if v.volumeInfo.Version != 0 {
		v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)
	}
	return v.SuperBlock.Version
}

// UpdateReplicaPlacement records rp in the live volume — the in-memory .vif
// copy and the superblock — and rewrites the .dat header when the data file
// is local. Keeps a mounted volume consistent with the .vif the caller just
// configured on disk, so heartbeats and later compactions report and rewrite
// the same placement.
func (v *Volume) UpdateReplicaPlacement(rp *super_block.ReplicaPlacement) error {
	v.dataFileSyncLock.Lock()
	defer v.dataFileSyncLock.Unlock()
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()
	v.superBlockAccessLock.Lock()
	defer v.superBlockAccessLock.Unlock()
	v.volumeInfoRWLock.Lock()
	defer v.volumeInfoRWLock.Unlock()

	v.SuperBlock.ReplicaPlacement = rp
	if v.volumeInfo != nil {
		v.volumeInfo.Replication = rp.String()
	}
	diskFile, ok := v.DataBackend.(*backend.DiskFile)
	if !ok {
		return nil // remote-tiered: the .vif carries the placement
	}
	if _, err := diskFile.WriteAt(v.SuperBlock.Bytes(), 0); err != nil {
		return fmt.Errorf("volume %d failed to write superblock: %v", v.Id, err)
	}
	return diskFile.Sync()
}

func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()

	if v.DataBackend == nil {
		return
	}

	datFileSize, modTime, e := v.DataBackend.GetStat()
	if e == nil {
		return uint64(datFileSize), v.nm.IndexFileSize(), modTime
	}
	glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
	return // -1 causes integer overflow and the volume to become unwritable.
}

// datAllocatedSize returns the disk space held by the .dat file, or 0 when it is unknown.
func (v *Volume) datAllocatedSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()

	df, ok := v.DataBackend.(*backend.DiskFile)
	if !ok {
		return 0
	}
	fi, err := df.File.Stat()
	if err != nil {
		return 0
	}
	return AllocatedSize(fi)
}

func (v *Volume) ContentSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.ContentSize()
}

func (v *Volume) doIsEmpty() (bool, error) {
	// check v.DataBackend.GetStat()
	if v.DataBackend == nil {
		return false, fmt.Errorf("v.DataBackend is nil")
	} else {
		datFileSize, _, e := v.DataBackend.GetStat()
		if e != nil {
			glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
			return false, fmt.Errorf("v.DataBackend.GetStat(): %v", e)
		}
		if datFileSize > super_block.SuperBlockSize {
			return false, nil
		}
	}
	// check v.nm.ContentSize()
	if v.nm != nil {
		if v.nm.ContentSize() > 0 {
			return false, nil
		}
	}
	return true, nil
}

// doIsGarbage reports whether every byte ever written is already deleted —
// the same all-garbage state vacuum measures, checked against the byte
// counters because the count counters drift on reload.
func (v *Volume) doIsGarbage() bool {
	if v.nm == nil {
		return false
	}
	contentSize := v.nm.ContentSize()
	return contentSize > 0 && v.nm.DeletedSize() >= contentSize
}

func (v *Volume) DeletedSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.DeletedSize()
}

func (v *Volume) FileCount() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return uint64(v.nm.FileCount())
}

func (v *Volume) DeletedCount() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return uint64(v.nm.DeletedCount())
}

func (v *Volume) MaxFileKey() types.NeedleId {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.MaxFileKey()
}

func (v *Volume) IndexFileSize() uint64 {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()
	if v.nm == nil {
		return 0
	}
	return v.nm.IndexFileSize()
}

func (v *Volume) DiskType() types.DiskType {
	return v.location.DiskType
}

func (v *Volume) SyncToDisk() {
	v.dataFileSyncLock.Lock()
	defer v.dataFileSyncLock.Unlock()
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()
	// Flush with the access lock dropped so readers are not stalled by the
	// fsync; writers and file swaps stay excluded by dataFileSyncLock.
	nm, backend := v.nm, v.DataBackend
	v.dataFileAccessLock.Unlock()
	if nm != nil {
		if err := nm.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
		}
	}
	if backend != nil {
		if err := backend.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
		}
	}
	v.dataFileAccessLock.Lock()
}

// Close cleanly shuts down this volume
func (v *Volume) Close() {
	// Wait for any in-progress compaction to finish and claim the flag so no
	// new compaction can start. This must happen BEFORE acquiring
	// dataFileSyncLock and dataFileAccessLock to avoid deadlocking with
	// CommitCompact which holds the flag while waiting for the locks.
	for !v.isCompactionInProgress.CompareAndSwap(false, true) {
		time.Sleep(521 * time.Millisecond)
		glog.Warningf("Volume Close wait for compaction %d", v.Id)
	}
	defer v.isCompactionInProgress.Store(false)

	v.dataFileSyncLock.Lock()
	defer v.dataFileSyncLock.Unlock()
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()

	v.doClose()
}

// SwapDataBackend atomically replaces the data backend and updates the
// remote-tier flag under dataFileAccessLock, closing the old backend. Both tier
// directions go through here so hasRemoteFile always matches the live backend:
// tier-down passes hasRemoteFile=false (now serving a local .dat), tier-up
// passes true. Keeping the swap and the flag under one lock means the heartbeat
// never observes a half-swapped backend or a flag that disagrees with it — a
// stale-false flag would make doDeleteRequest skip the new .dat's tombstones and
// disable the phantom-.dat guard.
func (v *Volume) SwapDataBackend(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
	v.dataFileSyncLock.Lock()
	defer v.dataFileSyncLock.Unlock()
	v.dataFileAccessLock.Lock()
	defer v.dataFileAccessLock.Unlock()
	v.swapDataBackendLocked(newBackend, hasRemoteFile)
}

// swapDataBackendLocked is the body of SwapDataBackend for callers that already
// hold dataFileAccessLock (e.g. load() reached while CommitCompact holds the
// lock). Reusing it from those under-lock paths avoids re-entering the
// non-reentrant lock, which would deadlock.
func (v *Volume) swapDataBackendLocked(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
	if v.DataBackend != nil {
		v.DataBackend.Close()
	}
	v.DataBackend = newBackend
	v.hasRemoteFile.Store(hasRemoteFile)
}

func (v *Volume) doClose() {
	if v.nm != nil {
		if err := v.nm.Sync(); err != nil {
			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
		}
		v.nm.Close()
		v.nm = nil
	}
	if v.DataBackend != nil {
		if err := v.DataBackend.Close(); err != nil {
			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
		}
		v.DataBackend = nil
		stats.VolumeServerVolumeGauge.WithLabelValues(v.Collection, "volume").Dec()
	}
}

func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaPlacement.GetCopyCount() > 1
}

// volume is expired if modified time + volume ttl < now
// except when volume is empty
// or when the volume does not have a ttl
// or when volumeSizeLimit is 0 when server just starts
func (v *Volume) expired(contentSize uint64, volumeSizeLimit uint64) bool {
	if volumeSizeLimit == 0 {
		// skip if we don't know size limit
		return false
	}
	if contentSize <= super_block.SuperBlockSize {
		return false
	}
	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
		return false
	}
	glog.V(2).Infof("volume %d now:%v lastModified:%v", v.Id, time.Now().Unix(), v.lastModifiedTsSeconds)
	livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTsSeconds)) / 60
	glog.V(2).Infof("volume %d ttl:%v lived:%v", v.Id, v.Ttl, livedMinutes)
	if int64(v.Ttl.Minutes()) < livedMinutes {
		return true
	}
	return false
}

// ExpireAtSec is when this volume's data becomes garbage, counted from its last
// write. Counting from the current time instead let every .vif rewrite -- a
// read-only mark, a tier upload, an EC encode -- hand an already expiring volume
// another full TTL. Zero when the volume has no TTL.
func (v *Volume) ExpireAtSec() uint64 {
	if v.Ttl == nil {
		return 0
	}
	ttlSeconds := v.Ttl.ToSeconds()
	if ttlSeconds == 0 {
		return 0
	}
	lastWriteSec := v.lastModifiedTsSeconds
	if lastWriteSec == 0 {
		lastWriteSec = uint64(time.Now().Unix())
	}
	return lastWriteSec + ttlSeconds
}

// wait either maxDelayMinutes or 10% of ttl minutes
func (v *Volume) expiredLongEnough(maxDelayMinutes uint32) bool {
	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
		return false
	}
	removalDelay := v.Ttl.Minutes() / 10
	if removalDelay > maxDelayMinutes {
		removalDelay = maxDelayMinutes
	}

	if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTsSeconds < uint64(time.Now().Unix()) {
		return true
	}
	return false
}

func (v *Volume) collectStatus() (maxFileKey types.NeedleId, datFileSize int64, modTime time.Time, fileCount, deletedCount, deletedSize uint64, ok bool) {
	v.dataFileAccessLock.RLock()
	defer v.dataFileAccessLock.RUnlock()

	if v.nm == nil || v.DataBackend == nil {
		return
	}

	ok = true

	maxFileKey = v.nm.MaxFileKey()
	datFileSize, modTime, _ = v.DataBackend.GetStat()
	fileCount = uint64(v.nm.FileCount())
	deletedCount = uint64(v.nm.DeletedCount())
	deletedSize = v.nm.DeletedSize()

	return
}

// ToVolumeInformationMessage fills into with what the master is told about this
// volume, allocating a message when into is nil. A heartbeat that keeps only
// the volumes it reports fills the same message for all the rest.
func (v *Volume) ToVolumeInformationMessage(into *master_pb.VolumeInformationMessage) (types.NeedleId, *master_pb.VolumeInformationMessage) {

	maxFileKey, volumeSize, modTime, fileCount, deletedCount, deletedSize, ok := v.collectStatus()

	if !ok {
		return 0, nil
	}

	// Detect phantom volumes: the .dat was unlinked from disk but is still held
	// open as a deleted FD, so the volume keeps serving and heartbeating while no
	// disk-path operation can ever succeed. Skip remote-tiered volumes, whose .dat
	// legitimately lives in cloud storage. Only a present .dat is cached for 30s; a
	// missing one is re-checked every heartbeat so the volume stays suppressed until
	// the file returns.
	if fileCount > 0 && !v.HasRemoteFile() {
		const diskCheckIntervalNs = 30 * int64(time.Second)
		now := time.Now().UnixNano()
		if now-v.lastDiskCheckNs.Load() > diskCheckIntervalNs {
			if _, err := os.Stat(v.FileName(".dat")); os.IsNotExist(err) {
				glog.Warningf("Volume %d: data file %s missing (held open as deleted FD) - not reporting to master", v.Id, v.FileName(".dat"))
				return 0, nil
			}
			v.lastDiskCheckNs.Store(now)
		}
	}

	volumeInfo := into
	if volumeInfo == nil {
		volumeInfo = &master_pb.VolumeInformationMessage{}
	}
	volumeInfo.Id = uint32(v.Id)
	volumeInfo.Size = uint64(volumeSize)
	volumeInfo.Collection = v.Collection
	volumeInfo.FileCount = fileCount
	volumeInfo.DeleteCount = deletedCount
	volumeInfo.DeletedByteCount = deletedSize
	volumeInfo.ReadOnly, _, volumeInfo.ReadOnlyCanDelete, _ = v.ReadOnlyReasons()
	volumeInfo.ReplicaPlacement = uint32(v.ReplicaPlacement.Byte())
	volumeInfo.Version = uint32(v.Version())
	volumeInfo.Ttl = v.Ttl.ToUint32()
	volumeInfo.CompactRevision = uint32(v.SuperBlock.CompactionRevision)
	volumeInfo.ModifiedAtSecond = modTime.Unix()
	volumeInfo.DiskType = string(v.location.DiskType)
	volumeInfo.DiskId = v.diskId
	volumeInfo.RemoteStorageName, volumeInfo.RemoteStorageKey = v.RemoteStorageNameKey()

	return maxFileKey, volumeInfo
}

func (v *Volume) RemoteStorageNameKey() (storageName, storageKey string) {
	if v.volumeInfo == nil {
		return
	}
	if len(v.volumeInfo.GetFiles()) == 0 {
		return
	}
	return v.volumeInfo.GetFiles()[0].BackendName(), v.volumeInfo.GetFiles()[0].GetKey()
}

func (v *Volume) IsReadOnly() bool {
	readOnly, _, _, _ := v.ReadOnlyReasons()
	return readOnly
}

// ReadOnlyReasons reports whether the volume refuses writes and why, reading the
// flags once so the reasons cannot disagree with the verdict.
func (v *Volume) ReadOnlyReasons() (readOnly, noWriteOrDelete, noWriteCanDelete, diskSpaceLow bool) {
	v.noWriteLock.RLock()
	noWriteOrDelete, noWriteCanDelete = v.noWriteOrDelete, v.noWriteCanDelete
	if v.ioUnavailable {
		noWriteOrDelete = true
	}
	v.noWriteLock.RUnlock()
	// The location is attached when the volume joins a disk location, which is
	// after NewVolume hands it back.
	diskSpaceLow = v.location != nil && v.location.isDiskSpaceLow.Load()
	return noWriteOrDelete || noWriteCanDelete || diskSpaceLow, noWriteOrDelete, noWriteCanDelete, diskSpaceLow
}

var errVolumeUnavailable = errors.New("volume unavailable")

func (v *Volume) UnavailableError() error {
	v.noWriteLock.RLock()
	unavailable := v.ioUnavailable
	reason := v.ioUnavailableError
	v.noWriteLock.RUnlock()
	if !unavailable {
		return nil
	}
	if reason == "" {
		return fmt.Errorf("volume %d is unavailable: %w", v.Id, errVolumeUnavailable)
	}
	return fmt.Errorf("volume %d is unavailable: %s: %w", v.Id, reason, errVolumeUnavailable)
}

func (v *Volume) markIoUnavailable(err error) {
	v.noWriteLock.Lock()
	v.noWriteOrDelete = true
	v.ioUnavailable = true
	v.ioUnavailableError = err.Error()
	v.noWriteLock.Unlock()
	v.markIoQuarantined()

	if persistErr := v.persistUnavailable(err.Error()); persistErr != nil {
		glog.Warningf("volume %d: failed to persist unavailable marker: %v", v.Id, persistErr)
	}
	if v.volumeInfo != nil {
		if persistErr := v.PersistReadOnly(true, false); persistErr != nil {
			glog.Warningf("volume %d: failed to persist unavailable state: %v", v.Id, persistErr)
		}
	}
	glog.Errorf("volume %d entered unavailable state after failed recovery: %v", v.Id, err)
}

// persistUnavailable records the failed-recovery state so a reload keeps the
// volume unavailable instead of serving an unverified .dat/index pair.
func (v *Volume) persistUnavailable(reason string) error {
	marker := v.FileName(".unavailable")
	f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	if _, err := f.WriteString(reason); err != nil {
		f.Close()
		return err
	}
	if err := f.Sync(); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	return util.FsyncDir(v.dir)
}

// restoreUnavailable re-arms the in-memory state the .unavailable marker
// recorded. The marker is deleted manually once the volume pair is verified.
func (v *Volume) restoreUnavailable() {
	reason, err := os.ReadFile(v.FileName(".unavailable"))
	if err != nil {
		return
	}
	v.noWriteLock.Lock()
	v.noWriteOrDelete = true
	v.ioUnavailable = true
	v.ioUnavailableError = strings.TrimSpace(string(reason))
	v.noWriteLock.Unlock()
	v.markIoQuarantined()
	glog.Warningf("volume %d is unavailable: %s", v.Id, v.ioUnavailableError)
}

func (v *Volume) PersistReadOnly(readOnly bool, canDelete bool) error {
	v.volumeInfoRWLock.Lock()
	defer v.volumeInfoRWLock.Unlock()
	prevReadOnly := v.volumeInfo.ReadOnly
	prevReadOnlyCanDelete := v.volumeInfo.ReadOnlyCanDelete
	v.volumeInfo.ReadOnly = readOnly
	v.volumeInfo.ReadOnlyCanDelete = readOnly && canDelete
	if err := v.SaveVolumeInfo(); err != nil {
		// A pre-commit failure (write/sync/close/rename) leaves the old
		// .vif intact, so roll back in-memory state to match it. A
		// NotCrashDurableError means the rename already committed the
		// new mode to disk; rolling back would split in-memory state
		// from the durable file, so keep the new state and propagate.
		var ndErr *volume_info.NotCrashDurableError
		if !errors.As(err, &ndErr) {
			v.volumeInfo.ReadOnly = prevReadOnly
			v.volumeInfo.ReadOnlyCanDelete = prevReadOnlyCanDelete
		}
		return fmt.Errorf("persist volume read-only state: %w", err)
	}
	return nil
}