diff --git a/seaweed-volume/src/server/heartbeat.rs b/seaweed-volume/src/server/heartbeat.rs index 9d56440a75..9c8153fb54 100644 --- a/seaweed-volume/src/server/heartbeat.rs +++ b/seaweed-volume/src/server/heartbeat.rs @@ -1190,7 +1190,7 @@ fn build_heartbeat_with_ec_status( for (disk_id, loc) in store.locations.iter().enumerate() { let disk_type_str = loc.disk_type.to_string(); let mut effective_max_count = loc.max_volume_count.load(Ordering::Relaxed); - if loc.is_disk_space_low.load(Ordering::Relaxed) { + if loc.is_disk_space_low.load(Ordering::Relaxed) || loc.read_only_mount { let used_slots = loc.volumes_len() as i32 + loc .ec_shard_count() diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index 9c548c6355..ab83c246fd 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -10,7 +10,7 @@ use std::io; use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; -use tracing::warn; +use tracing::{info, warn}; use crate::config::MinFreeSpace; use crate::storage::erasure_coding::ec_bitrot::remove_bitrot_sidecars; @@ -40,6 +40,10 @@ pub struct DiskLocation { volumes: HashMap>, ec_volumes: HashMap, pub is_disk_space_low: Arc, + /// Go's `DiskLocation.readOnlyMount`: dir or idx_dir is on a read-only + /// filesystem. Volumes load read-only and the location offers no free + /// slots. Fixed at startup. + pub read_only_mount: bool, pub available_space: AtomicU64, /// Headroom promised to in-flight copies that statfs cannot see yet. /// Probes keep it out of available_space so they cannot re-spend it; a @@ -71,11 +75,19 @@ impl DiskLocation { fs::create_dir_all(idx_directory)?; idx_directory.to_string() }; + let read_only_mount = is_read_only_mount(directory) || is_read_only_mount(&idx_dir); + if read_only_mount { + info!( + "{} is on a read-only filesystem: its volumes load read-only, no new volumes", + directory + ); + } let directory_uuid = Self::generate_directory_uuid(directory)?; Ok(DiskLocation { directory: directory.to_string(), idx_directory: idx_dir, + read_only_mount, directory_uuid, disk_type, tags, @@ -102,8 +114,35 @@ impl DiskLocation { } let dir_uuid = uuid::Uuid::new_v4().to_string(); - fs::write(path, &dir_uuid)?; - Ok(dir_uuid) + match fs::write(&path, &dir_uuid) { + Ok(()) => Ok(dir_uuid), + // The uuid file cannot be written on a read-only mount; derive a + // stable one from the directory plus its device and host so + // restarts report the same id and independent disks at the same + // path differ. + Err(_) if is_read_only_mount(directory) => { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + directory.hash(&mut h); + if let Ok(meta) = fs::metadata(directory) { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + meta.dev().hash(&mut h); + } + #[cfg(not(unix))] + { + let _ = meta; + } + } + if let Ok(host) = std::env::var("HOSTNAME") { + host.hash(&mut h); + } + let derived = (h.finish() as u128) << 64 | h.finish() as u128; + Ok(uuid::Uuid::from_u128(derived).to_string()) + } + Err(e) => Err(e), + } } // ---- Volume management ---- @@ -720,6 +759,9 @@ impl DiskLocation { /// Matches Go's FindFreeLocation formula: /// free = ((MaxVolumeCount - VolumesLen()) * DataShardsCount - EcShardCount()) / DataShardsCount pub fn free_volume_count(&self) -> i32 { + if self.read_only_mount { + return 0; + } use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; let max = self.max_volume_count.load(Ordering::Relaxed); let free_count = (max as i64 - self.volumes.len() as i64) * DATA_SHARDS_COUNT as i64 @@ -1179,6 +1221,27 @@ impl DiskLocation { } } +/// Mirror of Go's `util.IsReadOnlyMount`: access(2) with W_OK answers EROFS +/// for a filesystem mounted read-only regardless of the permission bits, and +/// also for root, unlike a permission-bits check. +#[cfg(unix)] +pub(crate) fn is_read_only_mount(path: &str) -> bool { + use std::ffi::CString; + let c_path = match CString::new(path.as_bytes()) { + Ok(p) => p, + Err(_) => return false, + }; + if unsafe { libc::access(c_path.as_ptr(), libc::W_OK) } == 0 { + return false; + } + io::Error::last_os_error().raw_os_error() == Some(libc::EROFS) +} + +#[cfg(not(unix))] +pub(crate) fn is_read_only_mount(_path: &str) -> bool { + false +} + /// Get total and free disk space for a given path. /// Returns (total_bytes, free_bytes). pub fn get_disk_stats(path: &str) -> (u64, u64) { diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 8fb92cd005..1b389af366 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -295,7 +295,7 @@ impl Store { if effective_free <= 0 { continue; } - if loc.is_disk_space_low.load(Ordering::Relaxed) { + if loc.is_disk_space_low.load(Ordering::Relaxed) || loc.read_only_mount { continue; } if !has_room_for_new_volume( @@ -330,7 +330,7 @@ impl Store { let mut best: Option<(usize, i64)> = None; for (i, loc) in self.locations.iter().enumerate() { - if !pred(loc) || loc.is_disk_space_low.load(Ordering::Relaxed) { + if !pred(loc) || loc.is_disk_space_low.load(Ordering::Relaxed) || loc.read_only_mount { continue; } @@ -471,6 +471,14 @@ impl Store { let mut best: Option<(usize, u8, usize, i64)> = None; for (i, loc) in self.locations.iter().enumerate() { let owned = owned_ec_shard_count(loc, vid, shard_ids); + if loc.read_only_mount { + // A read-only owner still owns the shard: placing another + // copy on a writable sibling would register it on two disks. + if owned > 0 { + return None; + } + continue; + } let free = ec_free_shard_count(loc, data_shard_count); if owned == 0 { if loc.is_disk_space_low.load(Ordering::Relaxed) { diff --git a/seaweed-volume/src/storage/vid_stamp.rs b/seaweed-volume/src/storage/vid_stamp.rs index a98d4eaaf2..bace9d751d 100644 --- a/seaweed-volume/src/storage/vid_stamp.rs +++ b/seaweed-volume/src/storage/vid_stamp.rs @@ -95,6 +95,11 @@ impl Store { }; let mut failed = false; for loc in &self.locations { + // The stamp lives on the data directory, which stays writable + // when only the index directory is read-only. + if super::disk_location::is_read_only_mount(&loc.directory) { + continue; + } if let Err(e) = write_vid_stamp(Path::new(&loc.directory), &stamp) { tracing::warn!("write vid stamp in {}: {}", loc.directory, e); failed = true; diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index e7efffe908..c212d154c4 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -20,6 +20,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; +use crate::storage::disk_location::is_read_only_mount; use crate::storage::idx; use crate::storage::io::read_exact_at; use crate::storage::io_error::{IoErrorTracker, StreakMark}; @@ -1635,7 +1636,10 @@ impl Volume { Ok(file) => { io.dat_file = Some(file); } - Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { + Err(e) + if e.kind() == io::ErrorKind::PermissionDenied + || e.kind() == io::ErrorKind::ReadOnlyFilesystem => + { io.dat_file = Some(open_volume_file(OpenOptions::new().read(true), &dat_path)?); self.set_no_write_flag(NO_WRITE_OR_DELETE, true); } @@ -1717,12 +1721,24 @@ impl Volume { && !Path::new(&self.file_name(".idx")).exists() && self.current_dat_file_size_io(io)? > SUPER_BLOCK_SIZE as u64 { - self.rebuild_idx_file()?; - info!( - volume_id = self.id.0, - idx = %self.file_name(".idx"), - "rebuilt the index from the data file" - ); + match self.rebuild_idx_file() { + Ok(()) => info!( + volume_id = self.id.0, + idx = %self.file_name(".idx"), + "rebuilt the index from the data file" + ), + // The index dir refuses writes: the in-memory loader + // rebuilds the rows to a temp file instead of skipping + // the volume (mirrors Go's rebuiltIdx path). + Err(e) if is_read_only_mount(&self.dir_idx.read().unwrap()) => { + warn!( + volume_id = self.id.0, + error = %e, + "index dir is read-only; indexing .dat in memory" + ); + } + Err(e) => return Err(e), + } } // Recover rows that deletes on a tiered read-only volume overwrote @@ -1742,6 +1758,13 @@ impl Volume { ), } + // The permission bits still allow writing on a read-only mount; + // a writable .dat beside an index directory that cannot record + // the index loads as no_write_or_delete, same as Go. + if !self.no_write_flags().0 && is_read_only_mount(&self.dir_idx.read().unwrap()) { + self.set_no_write_flag(NO_WRITE_OR_DELETE, true); + } + // A crash between the .dat append and the .idx append leaves a // .dat tail no index entry covers. Re-index complete needles and // cut an incomplete remainder before the index loads, so the @@ -1959,18 +1982,28 @@ impl Volume { let nm = CompactNeedleMap::load_from_idx(&mut idx_file, self.version())?; io.nm = Some(NeedleMap::InMemory(nm)); } else { - // Missing .idx with existing .dat could orphan needles + // Missing .idx with existing .dat could orphan needles. On a + // read-only index dir the rows still come back, from a temp + // rebuild rather than the lost index (mirrors Go's + // rebuiltIdx path); writes stay blocked by no_write_or_delete. let dat_path = self.file_name(".dat"); - if Path::new(&dat_path).exists() { - let dat_size = fs::metadata(&dat_path).map(|m| m.len()).unwrap_or(0); + let dat_size = fs::metadata(&dat_path).map(|m| m.len()).unwrap_or(0); + if dat_size > SUPER_BLOCK_SIZE as u64 + && is_read_only_mount(&self.dir_idx.read().unwrap()) + { + let (mut tmp_file, tmp_path) = self.rebuild_idx_to_temp()?; + let result = CompactNeedleMap::load_from_idx(&mut tmp_file, self.version()); + let _ = fs::remove_file(&tmp_path); + io.nm = Some(NeedleMap::InMemory(result?)); + } else { if dat_size > SUPER_BLOCK_SIZE as u64 { warn!( volume_id = self.id.0, ".idx file missing but .dat exists with data; needles may be orphaned" ); } + io.nm = Some(NeedleMap::InMemory(CompactNeedleMap::new())); } - io.nm = Some(NeedleMap::InMemory(CompactNeedleMap::new())); } } else { // Open read-write (create if missing) diff --git a/seaweed-volume/src/storage/volume_idx_rebuild.rs b/seaweed-volume/src/storage/volume_idx_rebuild.rs index fc36fd5e8a..00533cd5d9 100644 --- a/seaweed-volume/src/storage/volume_idx_rebuild.rs +++ b/seaweed-volume/src/storage/volume_idx_rebuild.rs @@ -2,8 +2,8 @@ //! `weed/storage/volume_idx_rebuild.go`. use std::fs::{self, OpenOptions}; -use std::io::{BufWriter, Write}; -use std::path::Path; +use std::io::{self, BufWriter, Seek, Write}; +use std::path::{Path, PathBuf}; use crate::storage::idx; use crate::storage::needle::Needle; @@ -101,6 +101,41 @@ impl Volume { } result } + + /// Write the volume's .idx rows to a new temporary file and return it + /// rewound for reading, for a volume whose index directory cannot hold + /// the rebuilt index. Mirrors Go's `rebuildIdxToTempFile`; the caller + /// removes the file. + pub(crate) fn rebuild_idx_to_temp(&self) -> Result<(fs::File, PathBuf), VolumeError> { + let dat_path = self.file_name(".dat"); + let tmp_path = std::env::temp_dir().join(format!("weed-idx-{}", uuid::Uuid::new_v4())); + let build = || -> Result { + let tmp_file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&tmp_path)?; + let mut scanner = VolumeFileScanner4RebuildIdx { + writer: BufWriter::new(&tmp_file), + dat_size: fs::metadata(&dat_path)?.len() as i64, + version: self.version(), + stopped: false, + }; + scan_volume_file(&dat_path, &mut scanner)?; + scanner.writer.flush()?; + drop(scanner); + let mut tmp_file = tmp_file; + tmp_file.seek(io::SeekFrom::Start(0))?; + Ok(tmp_file) + }; + match build() { + Ok(f) => Ok((f, tmp_path)), + Err(e) => { + let _ = fs::remove_file(&tmp_path); + Err(e) + } + } + } } #[cfg(test)] diff --git a/weed/command/master.go b/weed/command/master.go index 359721eb22..93db5be139 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -2,13 +2,16 @@ package command import ( "context" + "crypto/sha1" "crypto/tls" "fmt" + "io/fs" "math/rand/v2" "net" "net/http" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -230,7 +233,19 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { } // start raftServer - metaDir := path.Join(*masterOption.metaFolder, fmt.Sprintf("m%d", *masterOption.port)) + metaFolder := util.ResolvePath(*masterOption.metaFolder) + metaDir := path.Join(metaFolder, fmt.Sprintf("m%d", *masterOption.port)) + if util.IsReadOnlyMount(metaFolder) { + // raft rewrites conf/state even to start. Run it from a private copy and + // allocate no volume ids, so nothing is lost when the copy is dropped. + roCopy, err := copyDirForReadOnly(metaDir) + if err != nil { + glog.Fatalf("-mdir %s is read-only and could not be copied: %v", *masterOption.metaFolder, err) + } + glog.V(0).Infof("-mdir %s is on a read-only filesystem: raft runs from %s, volume growth disabled", *masterOption.metaFolder, roCopy) + metaDir = roCopy + ms.Topo.SetMetaReadOnly(true) + } isSingleMaster := isSingleMasterMode(*masterOption.peers) @@ -242,7 +257,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"), Peers: masterPeers, ServerAddr: myMasterAddress, - DataDir: util.ResolvePath(metaDir), + DataDir: metaDir, Topo: ms.Topo, RaftResumeState: *masterOption.raftResumeState, SingleMaster: isSingleMaster, @@ -408,6 +423,78 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { } } +// copyDirForReadOnly copies the master's raft directory (a few KB) into a +// writable directory under os.TempDir keyed by src, and returns its path. +// A previous copy is reused so a restart keeps the votes and committed +// entries the frozen -mdir could not record; it is rebuilt only when the +// on-disk state is newer, i.e. the mount was writable in between. A missing +// src yields an empty copy. +func copyDirForReadOnly(src string) (string, error) { + // Key by the absolute path: a relative -mdir from different working + // directories must not collide on one copy. + abs, err := filepath.Abs(src) + if err != nil { + abs = src + } + sum := sha1.Sum([]byte(abs)) + dst := filepath.Join(os.TempDir(), fmt.Sprintf("weed-master-ro-%x", sum[:8])) + if fi, err := os.Stat(dst); err == nil && fi.IsDir() && + fi.Mode().Perm() == 0700 && util.DirOwnedBySelf(fi) { + if !newestMtime(dst).Before(newestMtime(src)) { + return dst, nil + } + os.RemoveAll(dst) + } + staging, err := os.MkdirTemp("", "weed-master-ro-") + if err != nil { + return "", err + } + err = filepath.WalkDir(src, func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + if p == src && os.IsNotExist(walkErr) { + return nil + } + return walkErr + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + target := filepath.Join(staging, rel) + if d.IsDir() { + return os.MkdirAll(target, 0700) + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + return os.WriteFile(target, data, 0600) + }) + if err != nil { + os.RemoveAll(staging) + return "", err + } + if err = os.Rename(staging, dst); err != nil { + return staging, nil + } + return dst, nil +} + +// newestMtime returns the latest modification time under dir, or the zero +// time for a missing or empty one. +func newestMtime(dir string) (latest time.Time) { + filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if info, statErr := d.Info(); statErr == nil && info.ModTime().After(latest) { + latest = info.ModTime() + } + return nil + }) + return latest +} + func isSingleMasterMode(peers string) bool { p := strings.ToLower(strings.TrimSpace(peers)) return p == "none" diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index e861db47e1..7a6771a5fc 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -93,8 +93,14 @@ func (store *LevelDB2Store) buildOptions() *opt.Options { func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { glog.Infof("filer store leveldb2 dir: %s", dir) os.MkdirAll(dir, 0755) - if err := weed_util.TestFolderWritable(dir); err != nil { - return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + if weed_util.IsReadOnlyMount(dir) { + glog.V(0).Infof("filer store %s is on a read-only filesystem, opening it read-only", dir) + store.ReadOnly = true + } + if !store.ReadOnly { + if err := weed_util.TestFolderWritable(dir); err != nil { + return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + } } opts := store.buildOptions() diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 0b8748347a..3111e2a5ad 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -15,6 +15,7 @@ import ( leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -36,6 +37,7 @@ type LevelDB3Store struct { dbs map[string]*leveldb.DB dbsLock sync.RWMutex ReadOnly bool + emptyDB *leveldb.DB blockCacheSizeMB int writeBufferSizeMB int @@ -104,8 +106,14 @@ func (store *LevelDB3Store) buildOptions(name string) *opt.Options { func (store *LevelDB3Store) initialize(dir string) (err error) { glog.Infof("filer store leveldb3 dir: %s", dir) os.MkdirAll(dir, 0755) - if err := weed_util.TestFolderWritable(dir); err != nil { - return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + if weed_util.IsReadOnlyMount(dir) { + glog.V(0).Infof("filer store %s is on a read-only filesystem, opening it read-only", dir) + store.ReadOnly = true + } + if !store.ReadOnly { + if err := weed_util.TestFolderWritable(dir); err != nil { + return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + } } store.dir = dir @@ -173,11 +181,32 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo store.dbsLock.RUnlock() + // A bucket with no db directory has no entries. Opening it read-only + // cannot create the directory, so serve its reads from an empty + // in-memory db; the store's write methods refuse mutations. + if store.ReadOnly && !weed_util.FolderExists(fmt.Sprintf("%s/%s", store.dir, bucket)) { + db, err := store.emptyBucketDB() + return db, bucket, shortPath, err + } + db, err := store.createDB(bucket) return db, bucket, shortPath, err } +func (store *LevelDB3Store) emptyBucketDB() (*leveldb.DB, error) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + if store.emptyDB == nil { + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + return nil, err + } + store.emptyDB = db + } + return store.emptyDB, nil +} + func (store *LevelDB3Store) createDB(bucket string) (*leveldb.DB, error) { store.dbsLock.Lock() @@ -232,6 +261,9 @@ func writeOptions(ctx context.Context) *opt.WriteOptions { func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + if store.ReadOnly { + return fmt.Errorf("filer store leveldb3 is read-only") + } db, _, shortPath, err := store.findDB(entry.FullPath, false) if err != nil { return fmt.Errorf("findDB %s : %v", entry.FullPath, err) @@ -299,6 +331,9 @@ func (store *LevelDB3Store) FindEntry(ctx context.Context, fullpath weed_util.Fu func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { + if store.ReadOnly { + return fmt.Errorf("filer store leveldb3 is read-only") + } db, _, shortPath, err := store.findDB(fullpath, false) if err != nil { return fmt.Errorf("findDB %s : %v", fullpath, err) @@ -317,6 +352,9 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util. func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { + if store.ReadOnly { + return fmt.Errorf("filer store leveldb3 is read-only") + } db, bucket, shortPath, err := store.findDB(fullpath, true) if err != nil { return fmt.Errorf("findDB %s : %v", fullpath, err) @@ -450,4 +488,7 @@ func (store *LevelDB3Store) Shutdown() { for _, db := range store.dbs { db.Close() } + if store.emptyDB != nil { + store.emptyDB.Close() + } } diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 56d6283a9c..72d8783785 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -62,8 +62,11 @@ type DiskLocation struct { isDiskSpaceLow atomic.Bool isDiskUnavailable atomic.Bool - syncBatch SyncBatchOption - closeCh chan struct{} + // readOnlyMount: dir or idxDir is on a read-only filesystem. Volumes load + // read-only and the location offers no free slots. Fixed at startup. + readOnlyMount bool + syncBatch SyncBatchOption + closeCh chan struct{} } func GenerateDirUuid(dir string) (dirUuidString string, err error) { @@ -105,9 +108,27 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree glog.Fatalf("cannot create idx dir %s: %v", idxDir, err) } } + readOnlyMount := util.IsReadOnlyMount(dir) || util.IsReadOnlyMount(idxDir) + if readOnlyMount { + glog.V(0).Infof("%s is on a read-only filesystem: its volumes load read-only, no new volumes", dir) + } dirUuid, err := GenerateDirUuid(dir) if err != nil { - glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err) + if !readOnlyMount { + glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err) + } + // The uuid file cannot be written on a read-only mount; derive a + // stable one from the directory plus its device and host so restarts + // report the same id and independent disks at the same path differ. + diskKey := dir + if fi, statErr := os.Stat(dir); statErr == nil { + diskKey = fmt.Sprintf("%s@%x", dir, util.FileDeviceId(fi)) + } + if host, hostErr := os.Hostname(); hostErr == nil { + diskKey = host + "|" + diskKey + } + dirUuid = uuid.NewSHA1(uuid.NameSpaceOID, []byte(diskKey)).String() + glog.V(0).Infof("%s has no stored uuid and is read-only; using derived uuid %s", dir, dirUuid) } // Defensive copy of tags to prevent external mutation var copiedTags []string @@ -124,6 +145,7 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, MinFreeSpace: minFreeSpace, + readOnlyMount: readOnlyMount, syncBatch: DefaultSyncBatchOption, } location.volumes = make(map[needle.VolumeId]*Volume) diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index d3607c6f20..c3b6f81311 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -221,7 +221,14 @@ func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collect default: return nil, fmt.Errorf("cannot open ec volume index %s.ecx (or %s.ecx): %w", localBaseFileName, sharedBaseFileName, os.ErrNotExist) } - if ev.ecxFile, err = backend.OpenVolumeFile(indexBaseFileName+".ecx", os.O_RDWR); err != nil { + // A mounted read-only index directory keeps every shard readable: open + // the sealed index read-only and the journal read-only or not at all. + indexDirReadOnly := util.IsReadOnlyMount(ev.ecxActualDir) + ecxFlag := os.O_RDWR + if indexDirReadOnly { + ecxFlag = os.O_RDONLY + } + if ev.ecxFile, err = backend.OpenVolumeFile(indexBaseFileName+".ecx", ecxFlag); err != nil { return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %w", indexBaseFileName, err) } ecxFi, statErr := ev.ecxFile.Stat() @@ -239,36 +246,56 @@ func newEcVolumeWith(diskType types.DiskType, dir string, dirIdx string, collect // final inode, and it stops any compaction from replacing the file under // this handle. ev.ecjHold = acquireEcjHold(indexBaseFileName + ".ecj") - if ev.ecjFile, err = openEcjFile(indexBaseFileName + ".ecj"); err != nil { + if indexDirReadOnly { + // The journal cannot be created or appended on a read-only mount. + // Seed the deleted set from whatever is there, then drop the handle: + // a later delete reports no writable journal through the nil check. + if ev.ecjFile, err = backend.OpenVolumeFile(indexBaseFileName+".ecj", os.O_RDONLY); err != nil { + if !os.IsNotExist(err) { + ev.Close() + return nil, fmt.Errorf("cannot read ec volume journal %s.ecj: %w", indexBaseFileName, err) + } + err = nil + } + } else if ev.ecjFile, err = openEcjFile(indexBaseFileName + ".ecj"); err != nil { ev.Close() return nil, fmt.Errorf("cannot open ec volume journal %s.ecj: %v", indexBaseFileName, err) } - if ecjFi, statErr := ev.ecjFile.Stat(); statErr == nil { - ev.ecjFileSize = ecjFi.Size() - } else { - glog.Warningf("stat ec volume journal %s.ecj: %v", indexBaseFileName, statErr) + if ev.ecjFile != nil { + if ecjFi, statErr := ev.ecjFile.Stat(); statErr == nil { + ev.ecjFileSize = ecjFi.Size() + } else { + glog.Warningf("stat ec volume journal %s.ecj: %v", indexBaseFileName, statErr) + } } // Truncate a torn tail before the loader runs: appends land at the // physical end, so a trailing partial record would misalign every later - // delete and lose it on the next mount. - if ragged := ev.ecjFileSize % int64(types.NeedleIdSize); ragged != 0 { - whole := ev.ecjFileSize - ragged - glog.Warningf("ec volume %d: truncating torn .ecj tail %d -> %d bytes", vid, ev.ecjFileSize, whole) - if truncErr := ev.ecjFile.Truncate(whole); truncErr != nil { - ev.Close() - return nil, fmt.Errorf("ec volume %d: repair torn .ecj tail: %w", vid, truncErr) + // delete and lose it on the next mount. A read-only mount cannot repair + // it; the loader already skips the partial record. + if !indexDirReadOnly { + if ragged := ev.ecjFileSize % int64(types.NeedleIdSize); ragged != 0 { + whole := ev.ecjFileSize - ragged + glog.Warningf("ec volume %d: truncating torn .ecj tail %d -> %d bytes", vid, ev.ecjFileSize, whole) + if truncErr := ev.ecjFile.Truncate(whole); truncErr != nil { + ev.Close() + return nil, fmt.Errorf("ec volume %d: repair torn .ecj tail: %w", vid, truncErr) + } + if syncErr := ev.ecjFile.Sync(); syncErr != nil { + ev.Close() + return nil, fmt.Errorf("ec volume %d: sync .ecj after tail repair: %w", vid, syncErr) + } + ev.ecjFileSize = whole } - if syncErr := ev.ecjFile.Sync(); syncErr != nil { - ev.Close() - return nil, fmt.Errorf("ec volume %d: sync .ecj after tail repair: %w", vid, syncErr) - } - ev.ecjFileSize = whole } ev.deletedNeedles = make(map[types.NeedleId]struct{}) loadErr := ev.loadDeletedNeedlesFromEcj(ecjOps.readAt) if loadErr != nil { glog.Warningf("ec volume %d: load deleted needles from .ecj: %v", vid, loadErr) } + if indexDirReadOnly && ev.ecjFile != nil { + ev.ecjFile.Close() + ev.ecjFile = nil + } // read volume info. Prefer .vif at the data dir (where shards live), but // fall back to the index dir when the data dir does not have one — the diff --git a/weed/storage/store.go b/weed/storage/store.go index 5916ee5fb6..21c5efe5a1 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -323,7 +323,7 @@ func (s *Store) FindFreeLocation(filterFn func(location *DiskLocation) bool, rep if filterFn != nil && !filterFn(location) { continue } - if location.isDiskSpaceLow.Load() { + if location.isDiskSpaceLow.Load() || location.readOnlyMount { continue } currentFreeCount := location.MaxVolumeCount - int32(location.VolumesLen()) @@ -476,7 +476,7 @@ func hasRoomForNewVolume(location *DiskLocation, preallocate int64) bool { // hasFreeDiskLocation checks if a disk location has free space func (s *Store) hasFreeDiskLocation(location *DiskLocation) bool { // Check if disk space is low first - if location.isDiskSpaceLow.Load() { + if location.isDiskSpaceLow.Load() || location.readOnlyMount { return false } @@ -592,7 +592,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { } var deleteVids []needle.VolumeId effectiveMaxCount := location.MaxVolumeCount - if location.isDiskSpaceLow.Load() { + if location.isDiskSpaceLow.Load() || location.readOnlyMount { usedSlots := int32(location.LocalVolumesLen()) usedSlots += int32(erasure_coding.VolumeSlots(int64(location.EcShardCount()))) effectiveMaxCount = usedSlots diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index b66592f2bd..6c306046b3 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -89,6 +89,14 @@ func (s *Store) FindEcShardTargetLocation(collection string, vid needle.VolumeId ) for _, loc := range s.Locations { owned := ownedEcShardCount(loc, vid, shardIds) + if loc.readOnlyMount { + // A read-only owner still owns the shard: placing another copy + // on a writable sibling would register it on two disks. + if owned > 0 { + return nil + } + continue + } freeCount := ecFreeShardCount(loc, dataShardCount) if owned == 0 { if loc.isDiskSpaceLow.Load() { diff --git a/weed/storage/vid_stamp.go b/weed/storage/vid_stamp.go index 5e304d87f3..9176ef7e03 100644 --- a/weed/storage/vid_stamp.go +++ b/weed/storage/vid_stamp.go @@ -104,6 +104,11 @@ func (s *Store) saveVidStamps() { } failed := false for _, location := range s.Locations { + // The stamp lives on the data directory, which stays writable when + // only IdxDirectory is read-only. + if util.IsReadOnlyMount(location.Directory) { + continue + } if err := writeVidStamp(location.Directory, stamp); err != nil { glog.Warningf("write vid stamp in %s: %v", location.Directory, err) failed = true diff --git a/weed/storage/volume_idx_rebuild.go b/weed/storage/volume_idx_rebuild.go index 214a99d323..b31ce4ea10 100644 --- a/weed/storage/volume_idx_rebuild.go +++ b/weed/storage/volume_idx_rebuild.go @@ -96,3 +96,30 @@ func (v *Volume) rebuildIdxFile() error { } return fsyncDir(filepath.Dir(idxFileName)) } + +// rebuildIdxToTempFile writes the volume's .idx rows to a new temporary file +// and returns it rewound for reading, for a volume whose index directory +// cannot hold the rebuilt index. The caller owns the file and its path. +func (v *Volume) rebuildIdxToTempFile() (*os.File, error) { + datSize, _, err := v.DataBackend.GetStat() + if err != nil { + return nil, fmt.Errorf("stat %s: %w", v.FileName(".dat"), err) + } + tmpFile, err := os.CreateTemp("", "weed-idx-*") + if err != nil { + return nil, err + } + writer := bufio.NewWriter(tmpFile) + scanner := &VolumeFileScanner4RebuildIdx{writer: writer, datSize: datSize, version: v.Version()} + if scanErr := ScanVolumeFileFrom(scanner.version, v.DataBackend, int64(v.SuperBlock.BlockSize()), scanner); scanErr != nil { + err = scanErr + } else if err = writer.Flush(); err == nil { + _, err = tmpFile.Seek(0, io.SeekStart) + } + if err != nil { + tmpFile.Close() + os.Remove(tmpFile.Name()) + return nil, fmt.Errorf("rebuild %s from %s: %w", v.FileName(".idx"), v.FileName(".dat"), err) + } + return tmpFile, nil +} diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index f237d33836..ae2fd07c1b 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -197,6 +197,11 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind if !canRead { return fmt.Errorf("cannot read Volume Data file %s", v.FileName(".dat")) } + // The permission bits still allow writing on a read-only mount; open the + // .dat read-only there so the volume loads as noWriteOrDelete. + if canWrite && util.IsReadOnlyMount(v.dir) { + canWrite = false + } var dataFile *os.File if canWrite { dataFile, err = backend.OpenVolumeFile(v.FileName(".dat"), os.O_RDWR|os.O_CREATE) @@ -254,6 +259,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind v.dirIdx = v.dir } } + var rebuiltIdx *os.File // check volume idx files if err := v.checkIdxFile(); err != nil { // A remote-tiered volume with a stray .vif but no .idx must not @@ -265,10 +271,23 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind // A changed -dir.idx leaves the new directory without an index. // The .dat still holds every row, so rebuild rather than exit. if rebuildErr := v.rebuildIdxFile(); rebuildErr != nil { - glog.Errorf("skip volume %d (idx: %s): %v", v.Id, v.FileName(".idx"), rebuildErr) - return fmt.Errorf("rebuild volume idx file %s: %w", v.FileName(".idx"), rebuildErr) + if !util.IsReadOnlyMount(v.dirIdx) { + glog.Errorf("skip volume %d (idx: %s): %v", v.Id, v.FileName(".idx"), rebuildErr) + return fmt.Errorf("rebuild volume idx file %s: %w", v.FileName(".idx"), rebuildErr) + } + // The index dir cannot take the rebuilt .idx: rebuild its + // rows to a temp file and index .dat in memory instead of + // skipping the volume. Writes stay blocked by noWriteOrDelete. + if rebuiltIdx, err = v.rebuildIdxToTempFile(); err != nil { + glog.Errorf("skip volume %d (idx: %s): %v", v.Id, v.FileName(".idx"), err) + return fmt.Errorf("rebuild volume idx file %s: %w", v.FileName(".idx"), err) + } + defer os.Remove(rebuiltIdx.Name()) + v.noWriteOrDelete = true + glog.V(0).Infof("volume %d: %s is missing on a read-only mount; indexing %s in memory", v.Id, v.FileName(".idx"), v.FileName(".dat")) + } else { + glog.V(0).Infof("volume %d: rebuilt %s from %s", v.Id, v.FileName(".idx"), v.FileName(".dat")) } - glog.V(0).Infof("volume %d: rebuilt %s from %s", v.Id, v.FileName(".idx"), v.FileName(".dat")) } // Recover rows that deletes on a tiered read-only volume overwrote at // the front of .idx. Best effort: a volume that cannot be repaired is @@ -278,8 +297,13 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind } else if restored > 0 { glog.V(0).Infof("volume %d: recovered %d overwritten %s rows from %s", v.Id, restored, v.FileName(".idx"), v.FileName(".dat")) } + if !v.noWriteOrDelete && util.IsReadOnlyMount(v.dirIdx) { + v.noWriteOrDelete = true + } var indexFile *os.File - if v.noWriteOrDelete { + if rebuiltIdx != nil { + indexFile = rebuiltIdx + } else if v.noWriteOrDelete { glog.V(0).Infoln("open to read file", v.FileName(".idx")) if indexFile, err = backend.OpenVolumeFile(v.FileName(".idx"), os.O_RDONLY); err != nil { return fmt.Errorf("cannot read Volume Index %s: %v", v.FileName(".idx"), err) @@ -315,7 +339,17 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind // to v.nm yields a non-nil interface over a nil receiver. Clear v.nm // and close indexFile so the defer cleanup keys off v.nm cleanly. if v.noWriteOrDelete || v.noWriteCanDelete { - if v.nm, err = NewSortedFileNeedleMap(v.IndexFileName(), indexFile, v.Version()); err != nil { + if util.IsReadOnlyMount(v.dirIdx) && !isSortedFileFresh(v.IndexFileName()+".sdx", indexFile) { + // No usable .sdx and none can be written: index .idx in memory. + // Writes and deletes stay blocked by noWriteOrDelete. + if v.nm, err = LoadCompactNeedleMap(indexFile, v.Version()); err != nil { + glog.V(0).Infof("loading index %s to memory error: %v", v.FileName(".idx"), err) + v.nm = nil + } + // The map keeps indexFile only for appends this volume + // refuses; release the descriptor like SortedFileNeedleMap. + indexFile.Close() + } else if v.nm, err = NewSortedFileNeedleMap(v.IndexFileName(), indexFile, v.Version()); err != nil { glog.V(0).Infof("loading sorted db %s error: %v", v.FileName(".sdx"), err) v.nm = nil indexFile.Close() diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 00c5f118bb..7165eba990 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -94,6 +94,7 @@ type Topology struct { growthFrozen atomic.Bool growthFrozenAt atomic.Int64 + metaReadOnly atomic.Bool freezeOnMismatch atomic.Bool stateLossRecoveryWait atomic.Int64 allocationNotBefore atomic.Int64 @@ -618,7 +619,16 @@ func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []* return nil } +// SetMetaReadOnly disables volume id allocation: -mdir is on a read-only +// filesystem and raft runs from a copy that does not survive a restart. +func (t *Topology) SetMetaReadOnly(readOnly bool) { + t.metaReadOnly.Store(readOnly) +} + func (t *Topology) NextVolumeId() (needle.VolumeId, error) { + if t.metaReadOnly.Load() { + return 0, fmt.Errorf("volume growth disabled: -mdir is on a read-only filesystem") + } if !t.IsLeaderAndCanRead() { return 0, fmt.Errorf("as leader can not read yet") diff --git a/weed/util/readonly_mount.go b/weed/util/readonly_mount.go new file mode 100644 index 0000000000..f65a931b48 --- /dev/null +++ b/weed/util/readonly_mount.go @@ -0,0 +1,34 @@ +//go:build linux || darwin || freebsd || netbsd || openbsd || solaris + +package util + +import ( + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +// IsReadOnlyMount reports whether path is on a filesystem mounted read-only. +// access(2) with W_OK answers EROFS for that case regardless of the permission +// bits, and also for root, unlike TestFolderWritable and CheckFile. +func IsReadOnlyMount(path string) bool { + return unix.Access(path, unix.W_OK) == unix.EROFS +} + +// DirOwnedBySelf reports whether the directory info belongs to the current +// user. Content under os.TempDir is only trusted when it does. +func DirOwnedBySelf(fi os.FileInfo) bool { + st, ok := fi.Sys().(*syscall.Stat_t) + return ok && int(st.Uid) == os.Getuid() +} + +// FileDeviceId returns the device the file lives on, for a stable disk +// identity across processes. +func FileDeviceId(fi os.FileInfo) uint64 { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + return uint64(st.Dev) +} diff --git a/weed/util/readonly_mount_other.go b/weed/util/readonly_mount_other.go new file mode 100644 index 0000000000..a0cb45af1d --- /dev/null +++ b/weed/util/readonly_mount_other.go @@ -0,0 +1,23 @@ +//go:build !(linux || darwin || freebsd || netbsd || openbsd || solaris) + +package util + +import "os" + +// IsReadOnlyMount reports whether path is on a filesystem mounted read-only. +// Not detected on this platform. +func IsReadOnlyMount(path string) bool { + return false +} + +// DirOwnedBySelf reports whether the directory info belongs to the current +// user. Unreachable here: read-only mounts are not detected on this platform. +func DirOwnedBySelf(fi os.FileInfo) bool { + return false +} + +// FileDeviceId returns zero on this platform; read-only mounts are not +// detected, so no derived uuid consumes it. +func FileDeviceId(fi os.FileInfo) uint64 { + return 0 +}