mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
master, volume, filer: start on a read-only mounted directory and serve existing data read-only (#12081)
* master, volume, filer: start on a read-only mounted directory and serve existing data read-only A read-only mount was treated as writable (permission bits only), so every service failed on its first write: the master panicked writing raft conf.tmp, the filer exited opening the leveldb LOCK, and the volume server skipped every volume while still offering the directory as free capacity. - util.IsReadOnlyMount: access(W_OK) == EROFS. - volume: on a read-only -dir take the existing O_RDONLY/noWriteOrDelete load, index .idx in memory when no .sdx exists, offer no free slots (same branches as isDiskSpaceLow), skip the vid stamp there. - master: on a read-only -mdir run raft from a private copy and refuse new volume ids. - filer: leveldb2/leveldb3 set their existing ReadOnly option. * volume: skip read-only mounts when picking EC shard copy targets A shard copy landing on a read-only mount fails at write time; offer only writable locations like the other placement paths do. * volume: mirror the read-only mount handling in the Rust volume server The .dat open only fell back to read-only on EACCES, so a real read-only mount (EROFS) still failed the load. The index side already degrades once no_write_or_delete is set. Add is_read_only_mount via access(W_OK) == EROFS, mark DiskLocation.read_only_mount, and mirror the Go checks: no free slots, heartbeat max = used, no vid-stamp writes, no EC shard copy targets, and a read-only index dir forces the volume read-only. * volume: write vid stamps while only the index dir is read-only * filer: skip the write-bit check when a leveldb store opens read-only * filer: serve reads of buckets with no leveldb3 dir on a read-only store * volume: derive a stable dir uuid when a read-only mount cannot store one * master: keep the read-only raft copy across restarts so votes survive * volume: index a missing .idx in memory on a read-only mount * volume: release the .idx handle after an in-memory read-only load * volume: mount EC shards read-only when their index dir is on a read-only mount * volume: mark the index dir read-only before the tail repair opens .idx * volume: write vid stamps while only the index dir is read-only (Rust) * volume: derive a stable dir uuid and report no free slots on a read-only mount (Rust) * volume: index a missing .idx in memory on a read-only mount (Rust) * volume: refuse an EC copy whose shard owner is on a read-only disk * master: key the read-only raft copy by the absolute -mdir Two masters sharing the same relative -mdir text from different working directories would have collided on one temp copy and shared raft state. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: mix device and host into the read-only dir uuid Deriving the uuid from the path alone let independent servers at the same mount point report one uuid, which RegisterUuids rejects. Hash in the device id and hostname so restarts stay stable but disks differ. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: vic6996-web <320442181+vic6996-web@users.noreply.github.com> Co-authored-by: user.email <chrislusf@users.noreply.github.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
19 files changed
+522
-54
No files matched your search
+89
-2
@@ -2,13 +2,16 @@ package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -230,7 +233,19 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
|
||||
// start raftServer
|
||||
metaDir := path.Join(*masterOption.metaFolder, fmt.Sprintf("m%d", *masterOption.port))
|
||||
metaFolder := util.ResolvePath(*masterOption.metaFolder)
|
||||
metaDir := path.Join(metaFolder, fmt.Sprintf("m%d", *masterOption.port))
|
||||
if util.IsReadOnlyMount(metaFolder) {
|
||||
// raft rewrites conf/state even to start. Run it from a private copy and
|
||||
// allocate no volume ids, so nothing is lost when the copy is dropped.
|
||||
roCopy, err := copyDirForReadOnly(metaDir)
|
||||
if err != nil {
|
||||
glog.Fatalf("-mdir %s is read-only and could not be copied: %v", *masterOption.metaFolder, err)
|
||||
}
|
||||
glog.V(0).Infof("-mdir %s is on a read-only filesystem: raft runs from %s, volume growth disabled", *masterOption.metaFolder, roCopy)
|
||||
metaDir = roCopy
|
||||
ms.Topo.SetMetaReadOnly(true)
|
||||
}
|
||||
|
||||
isSingleMaster := isSingleMasterMode(*masterOption.peers)
|
||||
|
||||
@@ -242,7 +257,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
|
||||
Peers: masterPeers,
|
||||
ServerAddr: myMasterAddress,
|
||||
DataDir: util.ResolvePath(metaDir),
|
||||
DataDir: metaDir,
|
||||
Topo: ms.Topo,
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
SingleMaster: isSingleMaster,
|
||||
@@ -408,6 +423,78 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
}
|
||||
|
||||
// copyDirForReadOnly copies the master's raft directory (a few KB) into a
|
||||
// writable directory under os.TempDir keyed by src, and returns its path.
|
||||
// A previous copy is reused so a restart keeps the votes and committed
|
||||
// entries the frozen -mdir could not record; it is rebuilt only when the
|
||||
// on-disk state is newer, i.e. the mount was writable in between. A missing
|
||||
// src yields an empty copy.
|
||||
func copyDirForReadOnly(src string) (string, error) {
|
||||
// Key by the absolute path: a relative -mdir from different working
|
||||
// directories must not collide on one copy.
|
||||
abs, err := filepath.Abs(src)
|
||||
if err != nil {
|
||||
abs = src
|
||||
}
|
||||
sum := sha1.Sum([]byte(abs))
|
||||
dst := filepath.Join(os.TempDir(), fmt.Sprintf("weed-master-ro-%x", sum[:8]))
|
||||
if fi, err := os.Stat(dst); err == nil && fi.IsDir() &&
|
||||
fi.Mode().Perm() == 0700 && util.DirOwnedBySelf(fi) {
|
||||
if !newestMtime(dst).Before(newestMtime(src)) {
|
||||
return dst, nil
|
||||
}
|
||||
os.RemoveAll(dst)
|
||||
}
|
||||
staging, err := os.MkdirTemp("", "weed-master-ro-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = filepath.WalkDir(src, func(p string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
if p == src && os.IsNotExist(walkErr) {
|
||||
return nil
|
||||
}
|
||||
return walkErr
|
||||
}
|
||||
rel, err := filepath.Rel(src, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := filepath.Join(staging, rel)
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(target, 0700)
|
||||
}
|
||||
data, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(target, data, 0600)
|
||||
})
|
||||
if err != nil {
|
||||
os.RemoveAll(staging)
|
||||
return "", err
|
||||
}
|
||||
if err = os.Rename(staging, dst); err != nil {
|
||||
return staging, nil
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// newestMtime returns the latest modification time under dir, or the zero
|
||||
// time for a missing or empty one.
|
||||
func newestMtime(dir string) (latest time.Time) {
|
||||
filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if info, statErr := d.Info(); statErr == nil && info.ModTime().After(latest) {
|
||||
latest = info.ModTime()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return latest
|
||||
}
|
||||
|
||||
func isSingleMasterMode(peers string) bool {
|
||||
p := strings.ToLower(strings.TrimSpace(peers))
|
||||
return p == "none"
|
||||
|
||||
Reference in new issue
Block a user