Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-10-30 17:00:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into codex-rust-volume-server-bootstrap

Browse Source 
# Conflicts:
#	.github/workflows/volume-server-integration-tests.yml
#	test/volume_server/framework/cluster.go
#	test/volume_server/framework/cluster_dual.go
#	test/volume_server/framework/volume_fixture.go
#	test/volume_server/grpc/data_stream_success_test.go
#	test/volume_server/http/public_cors_methods_test.go
#	test/volume_server/matrix/config_profiles.go
This commit is contained in:
user.email committed 2026-10-24 22:23:33 -07:00
2 parents 73e4173789 ec8f979350
commit 9436099d55
3609 files changed
+709333 -121315
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/volume_server/framework/cluster.go
+144 -191
View File
@@ -18,26 +18,44 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// goBuildTags names the build tags a server the harness compiles must carry to
// hold the same offsets as the test binary asking for it. A 4-byte server and a
// 5-byte one disagree about every .idx row and reject each other's .vif, and
// the mixed Go/Rust suites run both at once.
func goBuildTags() []string {
if types.OffsetSize == 5 {
return []string{"-tags", "5BytesOffset"}
}
return nil
}
const (
defaultWaitTimeout = 30 * time.Second
defaultWaitTick = 200 * time.Millisecond
testVolumeSizeLimitMB = 32
)
var (
weedBinaryOnce sync.Once
weedBinaryPath string
weedBinaryErr error
)
// Cluster is a lightweight SeaweedFS master + one volume server test harness.
type Cluster struct {
testingTB testing.TB
profile matrix.Profile
weedBinary string
volumeBinary string
baseDir string
configDir string
logsDir string
keepLogs bool
weedBinary string
baseDir string
configDir string
logsDir string
keepLogs bool
masterPort int
masterGrpcPort int
@@ -48,16 +66,29 @@ type Cluster struct {
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
volumeDataDirs []string
cleanupOnce sync.Once
}
// StartSingleVolumeCluster boots one master and one volume server.
func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
t.Helper()
return StartSingleVolumeClusterWithDataDirs(t, profile, 1)
}
weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
// StartSingleVolumeClusterWithDataDirs boots one master and one volume server
// with dataDirCount separate data directories (passed to -dir as a comma list).
// Each directory becomes its own DiskLocation on the volume server, letting
// tests exercise multi-disk EC placement paths.
func StartSingleVolumeClusterWithDataDirs(t testing.TB, profile matrix.Profile, dataDirCount int) *Cluster {
t.Helper()
if dataDirCount < 1 {
t.Fatalf("dataDirCount must be >= 1, got %d", dataDirCount)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("resolve server binaries: %v", err)
t.Fatalf("resolve weed binary: %v", err)
}
baseDir, keepLogs, err := newWorkDir()
@@ -68,8 +99,19 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
volumeDataDir := filepath.Join(baseDir, "volume")
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
volumeDataDirs := make([]string, dataDirCount)
// Single-dir layout stays at baseDir/volume so existing fixtures
// (CorruptDatFile etc.) that hardcode that path keep working. Only
// multi-dir clusters get the "volumeN" layout.
for i := 0; i < dataDirCount; i++ {
if dataDirCount == 1 {
volumeDataDirs[i] = filepath.Join(baseDir, "volume")
} else {
volumeDataDirs[i] = filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
}
}
setupDirs := append([]string{configDir, logsDir, masterDataDir}, volumeDataDirs...)
for _, dir := range setupDirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
@@ -79,21 +121,17 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
t.Fatalf("write security config: %v", err)
}
masterPort, masterGrpcPort, err := allocateMasterPortPair()
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
ports, err := allocatePorts(3)
miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &Cluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
volumeBinary: volumeBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
@@ -118,11 +156,12 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
if err = c.startVolume(volumeDataDir); err != nil {
if err = c.startVolume(volumeDataDirs); err != nil {
masterLog := c.tailLog("master.log")
c.Stop()
t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog)
}
c.volumeDataDirs = volumeDataDirs
if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
volumeLog := c.tailLog("volume.log")
c.Stop()
@@ -157,6 +196,38 @@ func (c *Cluster) Stop() {
})
}
// RestartVolumeServer kills the volume server and starts a new one against
// the same data dirs and ports. The master keeps running. The previous run's
// volume.log is moved to volume.log.previous so the first run's logs survive
// a second-run startup failure.
func (c *Cluster) RestartVolumeServer() {
c.testingTB.Helper()
stopProcess(c.volumeCmd)
c.volumeCmd = nil
// Rotate the log; absent on the first call after a clean start, which
// is fine — startVolume will create it. Any real filesystem failure
// surfaces immediately on the next os.Create in startVolume.
oldLog := filepath.Join(c.logsDir, "volume.log")
_ = os.Rename(oldLog, filepath.Join(c.logsDir, "volume.log.previous"))
if err := c.startVolume(c.volumeDataDirs); err != nil {
c.testingTB.Fatalf("restart volume server: %v", err)
}
if err := c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
c.testingTB.Fatalf("wait for volume admin readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
}
if err := c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
c.testingTB.Fatalf("wait for volume grpc readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
}
}
// StopVolumeServer kills the volume server but leaves the master and data
// dirs alone. Pair with RestartVolumeServer or Stop.
func (c *Cluster) StopVolumeServer() {
c.testingTB.Helper()
stopProcess(c.volumeCmd)
c.volumeCmd = nil
}
func (c *Cluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
if err != nil {
@@ -182,12 +253,16 @@ func (c *Cluster) startMaster(dataDir string) error {
return c.masterCmd.Start()
}
func (c *Cluster) startVolume(dataDir string) error {
func (c *Cluster) startVolume(dataDirs []string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log"))
if err != nil {
return err
}
maxPerDir := make([]string, len(dataDirs))
for i := range dataDirs {
maxPerDir[i] = "16"
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
@@ -195,12 +270,16 @@ func (c *Cluster) startVolume(dataDir string) error {
"-port=" + strconv.Itoa(c.volumePort),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
"-port.public=" + strconv.Itoa(c.volumePubPort),
"-dir=" + dataDir,
"-max=16",
"-dir=" + strings.Join(dataDirs, ","),
"-max=" + strings.Join(maxPerDir, ","),
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
// Integration tests deliberately exercise loopback S3 endpoints
// (the test rig boots weed-mini next to the volume server); allow
// the SSRF guard to be bypassed for them.
"-volume.allowUntrustedRemoteEndpoints",
}
if c.profile.InflightUploadTimeout > 0 {
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
@@ -212,7 +291,7 @@ func (c *Cluster) startVolume(dataDir string) error {
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
}
c.volumeCmd = exec.Command(c.volumeBinary, args...)
c.volumeCmd = exec.Command(c.weedBinary, args...)
c.volumeCmd.Dir = c.baseDir
c.volumeCmd.Stdout = logFile
c.volumeCmd.Stderr = logFile
@@ -268,73 +347,6 @@ func stopProcess(cmd *exec.Cmd) {
}
}
func allocatePorts(count int) ([]int, error) {
const minPort = 10000
const maxPort = 55535
const host = "127.0.0.1"
rangeSize := maxPort - minPort + 1
listeners := make([]net.Listener, 0, count)
ports := make([]int, 0, count)
seen := make(map[int]struct{}, count)
closeAll := func() {
for _, ll := range listeners {
_ = ll.Close()
}
}
startOffset := int(time.Now().UnixNano() % int64(rangeSize))
for i := 0; i < count; i++ {
found := false
for offset := 0; offset < rangeSize; offset++ {
port := minPort + (startOffset+offset)%rangeSize
if _, exists := seen[port]; exists {
continue
}
l, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(port)))
if err != nil {
continue
}
seen[port] = struct{}{}
listeners = append(listeners, l)
ports = append(ports, port)
found = true
startOffset = (startOffset + offset + 1) % rangeSize
break
}
if !found {
closeAll()
return nil, fmt.Errorf("unable to allocate %d ports within range [%d,%d]", count, minPort, maxPort)
}
}
for _, l := range listeners {
_ = l.Close()
}
return ports, nil
}
func allocateMasterPortPair() (int, int, error) {
for masterPort := 10000; masterPort <= 55535; masterPort++ {
masterGrpcPort := masterPort + 10000
l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
if err != nil {
continue
}
l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
if err != nil {
_ = l1.Close()
continue
}
_ = l2.Close()
_ = l1.Close()
return masterPort, masterGrpcPort, nil
}
return 0, 0, errors.New("unable to find available master port pair")
}
func newWorkDir() (dir string, keepLogs bool, err error) {
keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_")
@@ -359,7 +371,7 @@ func writeSecurityConfig(configDir string, profile matrix.Profile) error {
b.WriteString("\"\n")
b.WriteString("expires_after_seconds = 60\n")
}
if profile.AccessUI {
if profile.EnableUIAccess {
if b.Len() > 0 {
b.WriteString("\n")
}
@@ -381,111 +393,45 @@ func FindOrBuildWeedBinary() (string, error) {
return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
}
repoRoot, err := detectRepoRoot()
if err != nil {
return "", err
}
if candidate := filepath.Join(repoRoot, "weed", "weed"); isExecutableFile(candidate) {
return candidate, nil
}
binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
return "", fmt.Errorf("create binary directory %s: %w", binDir, err)
}
binPath := filepath.Join(binDir, "weed")
if isExecutableFile(binPath) {
return binPath, nil
}
cmd := exec.Command("go", "build", "-o", binPath, ".")
cmd.Dir = filepath.Join(repoRoot, "weed")
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
}
if !isExecutableFile(binPath) {
return "", fmt.Errorf("built weed binary is not executable: %s", binPath)
}
return binPath, nil
}
// FindOrBuildServerBinaries returns master and volume executables.
// Master always runs from the Go weed binary; volume can be switched via env.
func FindOrBuildServerBinaries() (masterBinary string, volumeBinary string, err error) {
masterBinary, err = FindOrBuildWeedBinary()
if err != nil {
return "", "", err
}
volumeBinary, err = FindOrBuildVolumeServerBinary(masterBinary)
if err != nil {
return "", "", err
}
return masterBinary, volumeBinary, nil
}
// FindOrBuildVolumeServerBinary resolves the executable used for volume-server processes.
//
// Behavior:
// - `VOLUME_SERVER_BINARY=/path/to/bin`: use explicit executable path.
// - `VOLUME_SERVER_IMPL=rust`: build/use Rust volume server launcher.
// - default: use the same Go `weed` binary.
func FindOrBuildVolumeServerBinary(defaultBinary string) (string, error) {
if fromEnv := os.Getenv("VOLUME_SERVER_BINARY"); fromEnv != "" {
if isExecutableFile(fromEnv) {
return fromEnv, nil
weedBinaryOnce.Do(func() {
repoRoot := ""
if _, file, _, ok := runtime.Caller(0); ok {
repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
}
if repoRoot == "" {
weedBinaryErr = errors.New("unable to detect repository root")
return
}
return "", fmt.Errorf("VOLUME_SERVER_BINARY is set but not executable: %s", fromEnv)
}
impl := strings.ToLower(strings.TrimSpace(os.Getenv("VOLUME_SERVER_IMPL")))
if impl == "" || impl == "go" {
return defaultBinary, nil
}
if impl != "rust" {
return "", fmt.Errorf("unsupported VOLUME_SERVER_IMPL %q (supported: go, rust)", impl)
}
binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
weedBinaryErr = fmt.Errorf("create binary directory %s: %w", binDir, err)
return
}
binPath := filepath.Join(binDir, "weed")
repoRoot, err := detectRepoRoot()
if err != nil {
return "", err
}
return FindOrBuildRustVolumeServerBinary(repoRoot)
}
args := append([]string{"build"}, goBuildTags()...)
args = append(args, "-o", binPath, ".")
cmd := exec.Command("go", args...)
cmd.Dir = filepath.Join(repoRoot, "weed")
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
if err := cmd.Run(); err != nil {
weedBinaryErr = fmt.Errorf("build weed binary: %w\n%s", err, out.String())
return
}
if !isExecutableFile(binPath) {
weedBinaryErr = fmt.Errorf("built weed binary is not executable: %s", binPath)
return
}
weedBinaryPath = binPath
})
// FindOrBuildRustVolumeServerBinary builds the Rust volume server launcher when needed.
func FindOrBuildRustVolumeServerBinary(repoRoot string) (string, error) {
manifestPath := filepath.Join(repoRoot, "rust", "volume_server", "Cargo.toml")
if _, err := os.Stat(manifestPath); err != nil {
return "", fmt.Errorf("rust volume server manifest not found at %s: %w", manifestPath, err)
if weedBinaryErr != nil {
return "", weedBinaryErr
}
targetDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_rust_target")
binPath := filepath.Join(targetDir, "release", "weed-volume-rs")
if isExecutableFile(binPath) && os.Getenv("VOLUME_SERVER_RUST_REBUILD") != "1" {
return binPath, nil
}
cmd := exec.Command("cargo", "build", "--release", "--manifest-path", manifestPath, "--target-dir", targetDir)
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("build rust volume server binary: %w\n%s", err, out.String())
}
if !isExecutableFile(binPath) {
return "", fmt.Errorf("built rust volume server binary is not executable: %s", binPath)
}
return binPath, nil
}
func detectRepoRoot() (string, error) {
if _, file, _, ok := runtime.Caller(0); ok {
return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")), nil
}
return "", errors.New("unable to detect repository root")
return weedBinaryPath, nil
}
func isExecutableFile(path string) bool {
@@ -551,3 +497,10 @@ func (c *Cluster) VolumePublicURL() string {
func (c *Cluster) BaseDir() string {
return c.baseDir
}
// VolumeDataDirs returns the data directories the volume server was started with.
// Index 0 corresponds to DiskLocation 0, index 1 to DiskLocation 1, and so on.
// Tests can scan these directories to verify where files physically landed.
func (c *Cluster) VolumeDataDirs() []string {
return append([]string(nil), c.volumeDataDirs...)
}
test/volume_server/framework/cluster_dual.go
+7 -285
View File
@@ -1,295 +1,17 @@
package framework
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
type DualVolumeCluster struct {
testingTB testing.TB
profile matrix.Profile
// DualVolumeCluster is deprecated. Use MultiVolumeCluster instead.
// For backward compatibility, it's a type alias for MultiVolumeCluster.
type DualVolumeCluster = MultiVolumeCluster
weedBinary string
volumeBinary string
baseDir string
configDir string
logsDir string
keepLogs bool
masterPort int
masterGrpcPort int
volumePort0 int
volumeGrpcPort0 int
volumePubPort0 int
volumePort1 int
volumeGrpcPort1 int
volumePubPort1 int
masterCmd *exec.Cmd
volumeCmd0 *exec.Cmd
volumeCmd1 *exec.Cmd
cleanupOnce sync.Once
}
func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster {
t.Helper()
weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
if err != nil {
t.Fatalf("resolve server binaries: %v", err)
}
baseDir, keepLogs, err := newWorkDir()
if err != nil {
t.Fatalf("create temp test directory: %v", err)
}
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
volumeDataDir0 := filepath.Join(baseDir, "volume0")
volumeDataDir1 := filepath.Join(baseDir, "volume1")
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
}
if err = writeSecurityConfig(configDir, profile); err != nil {
t.Fatalf("write security config: %v", err)
}
masterPort, masterGrpcPort, err := allocateMasterPortPair()
if err != nil {
t.Fatalf("allocate master port pair: %v", err)
}
ports, err := allocatePorts(6)
if err != nil {
t.Fatalf("allocate volume ports: %v", err)
}
c := &DualVolumeCluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
volumeBinary: volumeBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
keepLogs: keepLogs,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePort0: ports[0],
volumeGrpcPort0: ports[1],
volumePubPort0: ports[0],
volumePort1: ports[2],
volumeGrpcPort1: ports[3],
volumePubPort1: ports[2],
}
if profile.SplitPublicPort {
c.volumePubPort0 = ports[4]
c.volumePubPort1 = ports[5]
}
if err = c.startMaster(masterDataDir); err != nil {
c.Stop()
t.Fatalf("start master: %v", err)
}
if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
masterLog := c.tailLog("master.log")
c.Stop()
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
if err = c.startVolume(0, volumeDataDir0); err != nil {
masterLog := c.tailLog("master.log")
c.Stop()
t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog)
}
if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil {
volumeLog := c.tailLog("volume0.log")
c.Stop()
t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil {
volumeLog := c.tailLog("volume0.log")
c.Stop()
t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
if err = c.startVolume(1, volumeDataDir1); err != nil {
volumeLog := c.tailLog("volume0.log")
c.Stop()
t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog)
}
if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil {
volumeLog := c.tailLog("volume1.log")
c.Stop()
t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil {
volumeLog := c.tailLog("volume1.log")
c.Stop()
t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
t.Cleanup(func() {
c.Stop()
})
return c
}
func (c *DualVolumeCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
stopProcess(c.volumeCmd1)
stopProcess(c.volumeCmd0)
stopProcess(c.masterCmd)
if !c.keepLogs && !c.testingTB.Failed() {
_ = os.RemoveAll(c.baseDir)
} else if c.baseDir != "" {
c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
}
})
}
func (c *DualVolumeCluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.masterPort),
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
"-mdir=" + dataDir,
"-peers=none",
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
"-defaultReplication=000",
}
c.masterCmd = exec.Command(c.weedBinary, args...)
c.masterCmd.Dir = c.baseDir
c.masterCmd.Stdout = logFile
c.masterCmd.Stderr = logFile
return c.masterCmd.Start()
}
func (c *DualVolumeCluster) startVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
volumePort := c.volumePort0
volumeGrpcPort := c.volumeGrpcPort0
volumePubPort := c.volumePubPort0
if index == 1 {
volumePort = c.volumePort1
volumeGrpcPort = c.volumeGrpcPort1
volumePubPort = c.volumePubPort1
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(volumePort),
"-port.grpc=" + strconv.Itoa(volumeGrpcPort),
"-port.public=" + strconv.Itoa(volumePubPort),
"-dir=" + dataDir,
"-max=16",
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
}
if c.profile.InflightUploadTimeout > 0 {
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
}
if c.profile.InflightDownloadTimeout > 0 {
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
}
cmd := exec.Command(c.volumeBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err = cmd.Start(); err != nil {
return err
}
if index == 1 {
c.volumeCmd1 = cmd
} else {
c.volumeCmd0 = cmd
}
return nil
}
func (c *DualVolumeCluster) waitForHTTP(url string) error {
return (&Cluster{}).waitForHTTP(url)
}
func (c *DualVolumeCluster) waitForTCP(addr string) error {
return (&Cluster{}).waitForTCP(addr)
}
func (c *DualVolumeCluster) tailLog(logName string) string {
return (&Cluster{logsDir: c.logsDir}).tailLog(logName)
}
func (c *DualVolumeCluster) MasterAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
}
func (c *DualVolumeCluster) MasterURL() string {
return "http://" + c.MasterAddress()
}
func (c *DualVolumeCluster) VolumeAdminAddress(index int) string {
if index == 1 {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1))
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0))
}
func (c *DualVolumeCluster) VolumePublicAddress(index int) string {
if index == 1 {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1))
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0))
}
func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string {
if index == 1 {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1))
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0))
}
func (c *DualVolumeCluster) VolumeAdminURL(index int) string {
return "http://" + c.VolumeAdminAddress(index)
}
func (c *DualVolumeCluster) VolumePublicURL(index int) string {
return "http://" + c.VolumePublicAddress(index)
// StartDualVolumeCluster starts a cluster with 2 volume servers.
// Deprecated: Use StartMultiVolumeClusterAuto(t, profile, 2) directly.
func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) MultiCluster {
return StartMultiVolumeClusterAuto(t, profile, 2)
}
test/volume_server/framework/cluster_interface.go
+63
View File
@@ -0,0 +1,63 @@
package framework
import (
"os"
"testing"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
// TestCluster is the common interface for single-volume cluster harnesses.
// Both *Cluster (Go volume) and *RustCluster (Rust volume) satisfy it.
type TestCluster interface {
MasterAddress() string
VolumeAdminAddress() string
VolumePublicAddress() string
VolumeGRPCAddress() string
VolumeServerAddress() string
MasterURL() string
VolumeAdminURL() string
VolumePublicURL() string
BaseDir() string
Stop()
}
func useRustVolumeServer() bool {
return os.Getenv("VOLUME_SERVER_IMPL") == "rust"
}
// StartVolumeCluster starts a single-volume cluster using either the Go or
// Rust volume server, depending on the VOLUME_SERVER_IMPL environment variable.
// Set VOLUME_SERVER_IMPL=rust to use the Rust volume server.
func StartVolumeCluster(t testing.TB, profile matrix.Profile) TestCluster {
t.Helper()
if useRustVolumeServer() {
return StartRustVolumeCluster(t, profile)
}
return StartSingleVolumeCluster(t, profile)
}
// MultiCluster is the common interface for multi-volume cluster harnesses.
// Both *MultiVolumeCluster (Go) and *RustMultiVolumeCluster (Rust) satisfy it.
type MultiCluster interface {
MasterAddress() string
MasterURL() string
BaseDir() string
VolumeAdminAddress(index int) string
VolumeAdminURL(index int) string
VolumePublicAddress(index int) string
VolumePublicURL(index int) string
VolumeGRPCAddress(index int) string
Stop()
}
// StartMultiVolumeClusterAuto starts a multi-volume cluster using either Go or
// Rust volume servers, depending on the VOLUME_SERVER_IMPL environment variable.
// Set VOLUME_SERVER_IMPL=rust to use Rust volume servers.
func StartMultiVolumeClusterAuto(t testing.TB, profile matrix.Profile, count int) MultiCluster {
t.Helper()
if useRustVolumeServer() {
return StartRustMultiVolumeCluster(t, profile, count)
}
return StartMultiVolumeCluster(t, profile, count)
}
test/volume_server/framework/cluster_interface_test.go
+20
View File
@@ -0,0 +1,20 @@
package framework
import "testing"
func TestUseRustVolumeServer(t *testing.T) {
t.Setenv("VOLUME_SERVER_IMPL", "rust")
if !useRustVolumeServer() {
t.Fatalf("expected rust selection when VOLUME_SERVER_IMPL=rust")
}
t.Setenv("VOLUME_SERVER_IMPL", "go")
if useRustVolumeServer() {
t.Fatalf("expected go selection when VOLUME_SERVER_IMPL=go")
}
t.Setenv("VOLUME_SERVER_IMPL", "")
if useRustVolumeServer() {
t.Fatalf("expected go selection when VOLUME_SERVER_IMPL is unset")
}
}
test/volume_server/framework/cluster_mixed.go
+371
View File
@@ -0,0 +1,371 @@
package framework
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
// for integration testing. Go servers occupy indices [0, goCount) and Rust
// servers occupy indices [goCount, goCount+rustCount).
type MixedVolumeCluster struct {
testingTB testing.TB
profile matrix.Profile
weedBinary string // Go weed binary (master + Go volume servers)
rustVolumeBinary string // Rust volume binary
baseDir string
configDir string
logsDir string
keepLogs bool
masterPort int
masterGrpcPort int
volumePorts []int
volumeGrpcPorts []int
volumePubPorts []int
isRust []bool // which servers are Rust
masterCmd *exec.Cmd
volumeCmds []*exec.Cmd
cleanupOnce sync.Once
}
// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
// servers, and rustCount Rust volume servers. Go servers come first in the index.
func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
t.Helper()
if goCount < 0 || rustCount < 0 {
t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
}
total := goCount + rustCount
if total < 2 {
t.Fatalf("need at least 2 volume servers, got %d", total)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("resolve weed binary: %v", err)
}
// Only build the Rust binary when Rust servers are requested.
var rustBinary string
if rustCount > 0 {
rustBinary, err = FindOrBuildRustBinary()
if err != nil {
t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
}
}
baseDir, keepLogs, err := newWorkDir()
if err != nil {
t.Fatalf("create temp test directory: %v", err)
}
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
dirs := []string{configDir, logsDir, masterDataDir}
for i := 0; i < total; i++ {
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
}
for _, dir := range dirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
}
if err = writeSecurityConfig(configDir, profile); err != nil {
t.Fatalf("write security config: %v", err)
}
// 2 ports per server (admin, grpc); add 1 more when public port is split out.
portsPerServer := 2
if profile.SplitPublicPort {
portsPerServer = 3
}
miniPorts, ports, err := testutil.AllocatePortSet(1, total*portsPerServer)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
isRust := make([]bool, total)
for i := goCount; i < total; i++ {
isRust[i] = true
}
c := &MixedVolumeCluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
rustVolumeBinary: rustBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
keepLogs: keepLogs,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePorts: make([]int, total),
volumeGrpcPorts: make([]int, total),
volumePubPorts: make([]int, total),
isRust: isRust,
volumeCmds: make([]*exec.Cmd, total),
}
for i := 0; i < total; i++ {
baseIdx := i * portsPerServer
c.volumePorts[i] = ports[baseIdx]
c.volumeGrpcPorts[i] = ports[baseIdx+1]
if profile.SplitPublicPort {
c.volumePubPorts[i] = ports[baseIdx+2]
} else {
c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
}
}
// Start master
if err = c.startMaster(masterDataDir); err != nil {
c.Stop()
t.Fatalf("start master: %v", err)
}
helper := &Cluster{logsDir: logsDir}
if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
masterLog := helper.tailLog("master.log")
c.Stop()
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
// Start volume servers
for i := 0; i < total; i++ {
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
if isRust[i] {
err = c.startRustVolume(i, volumeDataDir)
} else {
err = c.startGoVolume(i, volumeDataDir)
}
if err != nil {
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
c.Stop()
t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
}
// Rust uses /healthz, Go uses /status
healthURL := c.VolumeAdminURL(i) + "/status"
if isRust[i] {
healthURL = c.VolumeAdminURL(i) + "/healthz"
}
if err = helper.waitForHTTP(healthURL); err != nil {
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
c.Stop()
t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
}
if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
c.Stop()
t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
}
}
t.Cleanup(func() {
c.Stop()
})
return c
}
func (c *MixedVolumeCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
for i := len(c.volumeCmds) - 1; i >= 0; i-- {
stopProcess(c.volumeCmds[i])
}
stopProcess(c.masterCmd)
if !c.keepLogs && !c.testingTB.Failed() {
_ = os.RemoveAll(c.baseDir)
} else if c.baseDir != "" {
c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
}
})
}
func (c *MixedVolumeCluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.masterPort),
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
"-mdir=" + dataDir,
"-peers=none",
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
"-defaultReplication=000",
}
c.masterCmd = exec.Command(c.weedBinary, args...)
c.masterCmd.Dir = c.baseDir
c.masterCmd.Stdout = logFile
c.masterCmd.Stderr = logFile
if err = c.masterCmd.Start(); err != nil {
logFile.Close()
return err
}
logFile.Close() // child inherited the fd
return nil
}
func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.volumePorts[index]),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
"-dir=" + dataDir,
"-max=16",
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
"-volume.allowUntrustedRemoteEndpoints",
}
if c.profile.InflightUploadTimeout > 0 {
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
}
if c.profile.InflightDownloadTimeout > 0 {
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
}
if c.profile.FileSizeLimitMB > 0 {
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
}
cmd := exec.Command(c.weedBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err = cmd.Start(); err != nil {
logFile.Close()
return err
}
logFile.Close() // child inherited the fd
c.volumeCmds[index] = cmd
return nil
}
func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
args := rustVolumeArgs(
c.profile,
c.configDir,
c.masterPort,
c.volumePorts[index],
c.volumeGrpcPorts[index],
c.volumePubPorts[index],
dataDir,
)
cmd := exec.Command(c.rustVolumeBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err = cmd.Start(); err != nil {
logFile.Close()
return err
}
logFile.Close() // child inherited the fd
c.volumeCmds[index] = cmd
return nil
}
// --- accessor methods (mirror MultiVolumeCluster) ---
func (c *MixedVolumeCluster) MasterAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
}
func (c *MixedVolumeCluster) MasterURL() string {
return "http://" + c.MasterAddress()
}
func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
if index < 0 || index >= len(c.volumePorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
}
func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
if index < 0 || index >= len(c.volumePubPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
}
func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
if index < 0 || index >= len(c.volumeGrpcPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
}
func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
return "http://" + c.VolumeAdminAddress(index)
}
func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
return "http://" + c.VolumePublicAddress(index)
}
func (c *MixedVolumeCluster) BaseDir() string {
return c.baseDir
}
// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
if index < 0 || index >= len(c.volumePorts) {
return ""
}
return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
}
func (c *MixedVolumeCluster) IsRust(index int) bool {
if index < 0 || index >= len(c.isRust) {
return false
}
return c.isRust[index]
}
test/volume_server/framework/cluster_multi.go
+351
View File
@@ -0,0 +1,351 @@
package framework
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
type MultiVolumeCluster struct {
testingTB testing.TB
profile matrix.Profile
weedBinary string
baseDir string
configDir string
logsDir string
keepLogs bool
volumeServerCount int
disksPerServer int
masterPort int
masterGrpcPort int
volumePorts []int
volumeGrpcPorts []int
volumePubPorts []int
volumeDiskDirs [][]string // per server: the data directories (one DiskLocation each)
masterCmd *exec.Cmd
volumeCmds []*exec.Cmd
cleanupOnce sync.Once
}
// StartMultiVolumeCluster starts a cluster with serverCount volume servers, one
// data directory (DiskLocation) each.
func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *MultiVolumeCluster {
return StartMultiVolumeClusterWithDisks(t, profile, serverCount, 1)
}
// StartMultiVolumeClusterWithDisks starts serverCount volume servers, each with
// disksPerServer data directories passed to -dir as a comma list so every
// directory becomes its own DiskLocation. Lets tests exercise per-disk EC
// layouts (e.g. shards on one disk, a stub/index on a sibling disk).
func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serverCount, disksPerServer int) *MultiVolumeCluster {
t.Helper()
if serverCount < 1 {
t.Fatalf("serverCount must be at least 1, got %d", serverCount)
}
if disksPerServer < 1 {
t.Fatalf("disksPerServer must be at least 1, got %d", disksPerServer)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("resolve weed binary: %v", err)
}
baseDir, keepLogs, err := newWorkDir()
if err != nil {
t.Fatalf("create temp test directory: %v", err)
}
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
// Create directories for master and all volume servers. With one disk the
// layout stays at baseDir/volumeN; multi-disk servers get baseDir/volumeN/diskD.
dirs := []string{configDir, logsDir, masterDataDir}
volumeDiskDirs := make([][]string, serverCount)
for i := 0; i < serverCount; i++ {
serverDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
volumeDiskDirs[i] = make([]string, disksPerServer)
for d := 0; d < disksPerServer; d++ {
if disksPerServer == 1 {
volumeDiskDirs[i][d] = serverDir
} else {
volumeDiskDirs[i][d] = filepath.Join(serverDir, fmt.Sprintf("disk%d", d))
}
dirs = append(dirs, volumeDiskDirs[i][d])
}
}
for _, dir := range dirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
}
if err = writeSecurityConfig(configDir, profile); err != nil {
t.Fatalf("write security config: %v", err)
}
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
// If SplitPublicPort is true, we need an additional port per server
portsPerServer := 3
if profile.SplitPublicPort {
portsPerServer = 4
}
totalPorts := serverCount * portsPerServer
miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &MultiVolumeCluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
keepLogs: keepLogs,
volumeServerCount: serverCount,
disksPerServer: disksPerServer,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePorts: make([]int, serverCount),
volumeGrpcPorts: make([]int, serverCount),
volumePubPorts: make([]int, serverCount),
volumeDiskDirs: volumeDiskDirs,
volumeCmds: make([]*exec.Cmd, serverCount),
}
// Assign ports to each volume server
for i := 0; i < serverCount; i++ {
baseIdx := i * portsPerServer
c.volumePorts[i] = ports[baseIdx]
c.volumeGrpcPorts[i] = ports[baseIdx+1]
// Assign public port, using baseIdx+3 if SplitPublicPort, else baseIdx+2
pubPortIdx := baseIdx + 2
if profile.SplitPublicPort {
pubPortIdx = baseIdx + 3
}
c.volumePubPorts[i] = ports[pubPortIdx]
}
// Start master
if err = c.startMaster(masterDataDir); err != nil {
c.Stop()
t.Fatalf("start master: %v", err)
}
if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
masterLog := c.tailLog("master.log")
c.Stop()
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
// Start all volume servers
for i := 0; i < serverCount; i++ {
if err = c.startVolume(i, c.volumeDiskDirs[i]); err != nil {
// Log current server's log for debugging startup failures
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("start volume server %d: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
}
if err = c.waitForHTTP(c.VolumeAdminURL(i) + "/status"); err != nil {
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("wait for volume server %d readiness: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
}
if err = c.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("wait for volume server %d grpc readiness: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
}
}
t.Cleanup(func() {
c.Stop()
})
return c
}
// StartTripleVolumeCluster is a convenience wrapper that starts a cluster with 3 volume servers
func StartTripleVolumeCluster(t testing.TB, profile matrix.Profile) *MultiVolumeCluster {
return StartMultiVolumeCluster(t, profile, 3)
}
func (c *MultiVolumeCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
// Stop volume servers in reverse order
for i := len(c.volumeCmds) - 1; i >= 0; i-- {
stopProcess(c.volumeCmds[i])
}
stopProcess(c.masterCmd)
if !c.keepLogs && !c.testingTB.Failed() {
_ = os.RemoveAll(c.baseDir)
} else if c.baseDir != "" {
c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
}
})
}
func (c *MultiVolumeCluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.masterPort),
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
"-mdir=" + dataDir,
"-peers=none",
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
"-defaultReplication=000",
}
c.masterCmd = exec.Command(c.weedBinary, args...)
c.masterCmd.Dir = c.baseDir
c.masterCmd.Stdout = logFile
c.masterCmd.Stderr = logFile
return c.masterCmd.Start()
}
func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
maxPerDir := make([]string, len(dataDirs))
for i := range dataDirs {
maxPerDir[i] = "16"
}
args := []string{
"-config_dir=" + c.configDir,
"volume",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.volumePorts[index]),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
"-dir=" + strings.Join(dataDirs, ","),
"-max=" + strings.Join(maxPerDir, ","),
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
"-readMode=" + c.profile.ReadMode,
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
"-volume.allowUntrustedRemoteEndpoints",
}
if c.profile.InflightUploadTimeout > 0 {
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
}
if c.profile.InflightDownloadTimeout > 0 {
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
}
if c.profile.FileSizeLimitMB > 0 {
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
}
cmd := exec.Command(c.weedBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err = cmd.Start(); err != nil {
return err
}
c.volumeCmds[index] = cmd
return nil
}
func (c *MultiVolumeCluster) waitForHTTP(url string) error {
return (&Cluster{}).waitForHTTP(url)
}
func (c *MultiVolumeCluster) waitForTCP(addr string) error {
return (&Cluster{}).waitForTCP(addr)
}
func (c *MultiVolumeCluster) tailLog(logName string) string {
return (&Cluster{logsDir: c.logsDir}).tailLog(logName)
}
func (c *MultiVolumeCluster) MasterAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
}
func (c *MultiVolumeCluster) MasterURL() string {
return "http://" + c.MasterAddress()
}
func (c *MultiVolumeCluster) VolumeAdminAddress(index int) string {
if index < 0 || index >= len(c.volumePorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
}
func (c *MultiVolumeCluster) VolumePublicAddress(index int) string {
if index < 0 || index >= len(c.volumePubPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
}
func (c *MultiVolumeCluster) VolumeGRPCAddress(index int) string {
if index < 0 || index >= len(c.volumeGrpcPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
}
func (c *MultiVolumeCluster) VolumeAdminURL(index int) string {
return "http://" + c.VolumeAdminAddress(index)
}
func (c *MultiVolumeCluster) VolumePublicURL(index int) string {
return "http://" + c.VolumePublicAddress(index)
}
func (c *MultiVolumeCluster) BaseDir() string {
return c.baseDir
}
// VolumeDiskDir returns the data directory backing disk diskIndex on volume
// server serverIndex — i.e. the directory for that DiskLocation, for file
// inspection in tests.
func (c *MultiVolumeCluster) VolumeDiskDir(serverIndex, diskIndex int) string {
if serverIndex < 0 || serverIndex >= len(c.volumeDiskDirs) {
return ""
}
disks := c.volumeDiskDirs[serverIndex]
if diskIndex < 0 || diskIndex >= len(disks) {
return ""
}
return disks[diskIndex]
}
test/volume_server/framework/cluster_multi_rust.go
+287
View File
@@ -0,0 +1,287 @@
package framework
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
// RustMultiVolumeCluster wraps a Go master + multiple Rust volume servers
// for integration testing. It mirrors MultiVolumeCluster but uses the Rust
// volume binary instead of the Go weed binary for volume servers.
type RustMultiVolumeCluster struct {
testingTB testing.TB
profile matrix.Profile
weedBinary string // Go weed binary (for the master)
rustVolumeBinary string // Rust volume binary
baseDir string
configDir string
logsDir string
keepLogs bool
volumeServerCount int
masterPort int
masterGrpcPort int
volumePorts []int
volumeGrpcPorts []int
volumePubPorts []int
masterCmd *exec.Cmd
volumeCmds []*exec.Cmd
cleanupOnce sync.Once
}
// StartRustMultiVolumeCluster starts a cluster with a Go master and the
// specified number of Rust volume servers.
func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *RustMultiVolumeCluster {
t.Helper()
if serverCount < 1 {
t.Fatalf("serverCount must be at least 1, got %d", serverCount)
}
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("resolve weed binary: %v", err)
}
rustBinary, err := FindOrBuildRustBinary()
if err != nil {
t.Fatalf("resolve rust volume binary: %v", err)
}
baseDir, keepLogs, err := newWorkDir()
if err != nil {
t.Fatalf("create temp test directory: %v", err)
}
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
// Create directories for master and all volume servers
dirs := []string{configDir, logsDir, masterDataDir}
for i := 0; i < serverCount; i++ {
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
}
for _, dir := range dirs {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
}
if err = writeSecurityConfig(configDir, profile); err != nil {
t.Fatalf("write security config: %v", err)
}
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
// If SplitPublicPort is true, we need an additional port per server
portsPerServer := 3
if profile.SplitPublicPort {
portsPerServer = 4
}
totalPorts := serverCount * portsPerServer
miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
c := &RustMultiVolumeCluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
rustVolumeBinary: rustBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
keepLogs: keepLogs,
volumeServerCount: serverCount,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePorts: make([]int, serverCount),
volumeGrpcPorts: make([]int, serverCount),
volumePubPorts: make([]int, serverCount),
volumeCmds: make([]*exec.Cmd, serverCount),
}
// Assign ports to each volume server
for i := 0; i < serverCount; i++ {
baseIdx := i * portsPerServer
c.volumePorts[i] = ports[baseIdx]
c.volumeGrpcPorts[i] = ports[baseIdx+1]
// Assign public port, using baseIdx+3 if SplitPublicPort, else baseIdx+2
pubPortIdx := baseIdx + 2
if profile.SplitPublicPort {
pubPortIdx = baseIdx + 3
}
c.volumePubPorts[i] = ports[pubPortIdx]
}
// Start master (Go)
if err = c.startMaster(masterDataDir); err != nil {
c.Stop()
t.Fatalf("start master: %v", err)
}
helper := &Cluster{logsDir: logsDir}
if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
masterLog := helper.tailLog("master.log")
c.Stop()
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
// Start all Rust volume servers
for i := 0; i < serverCount; i++ {
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
if err = c.startRustVolume(i, volumeDataDir); err != nil {
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("start rust volume server %d: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
}
if err = helper.waitForHTTP(c.VolumeAdminURL(i) + "/healthz"); err != nil {
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("wait for rust volume server %d readiness: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
}
if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
volumeLog := fmt.Sprintf("volume%d.log", i)
c.Stop()
t.Fatalf("wait for rust volume server %d grpc readiness: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
}
}
t.Cleanup(func() {
c.Stop()
})
return c
}
func (c *RustMultiVolumeCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
// Stop volume servers in reverse order
for i := len(c.volumeCmds) - 1; i >= 0; i-- {
stopProcess(c.volumeCmds[i])
}
stopProcess(c.masterCmd)
if !c.keepLogs && !c.testingTB.Failed() {
_ = os.RemoveAll(c.baseDir)
} else if c.baseDir != "" {
c.testingTB.Logf("rust multi volume server integration logs kept at %s", c.baseDir)
}
})
}
func (c *RustMultiVolumeCluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(c.masterPort),
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
"-mdir=" + dataDir,
"-peers=none",
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
"-defaultReplication=000",
}
c.masterCmd = exec.Command(c.weedBinary, args...)
c.masterCmd.Dir = c.baseDir
c.masterCmd.Stdout = logFile
c.masterCmd.Stderr = logFile
return c.masterCmd.Start()
}
func (c *RustMultiVolumeCluster) startRustVolume(index int, dataDir string) error {
logName := fmt.Sprintf("volume%d.log", index)
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
if err != nil {
return err
}
args := rustVolumeArgs(
c.profile,
c.configDir,
c.masterPort,
c.volumePorts[index],
c.volumeGrpcPorts[index],
c.volumePubPorts[index],
dataDir,
)
cmd := exec.Command(c.rustVolumeBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err = cmd.Start(); err != nil {
return err
}
c.volumeCmds[index] = cmd
return nil
}
// --- accessor methods (mirror MultiVolumeCluster) ---
func (c *RustMultiVolumeCluster) MasterAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
}
func (c *RustMultiVolumeCluster) MasterURL() string {
return "http://" + c.MasterAddress()
}
func (c *RustMultiVolumeCluster) VolumeAdminAddress(index int) string {
if index < 0 || index >= len(c.volumePorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
}
func (c *RustMultiVolumeCluster) VolumePublicAddress(index int) string {
if index < 0 || index >= len(c.volumePubPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
}
func (c *RustMultiVolumeCluster) VolumeGRPCAddress(index int) string {
if index < 0 || index >= len(c.volumeGrpcPorts) {
return ""
}
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
}
func (c *RustMultiVolumeCluster) VolumeAdminURL(index int) string {
return "http://" + c.VolumeAdminAddress(index)
}
func (c *RustMultiVolumeCluster) VolumePublicURL(index int) string {
return "http://" + c.VolumePublicAddress(index)
}
func (c *RustMultiVolumeCluster) BaseDir() string {
return c.baseDir
}
test/volume_server/framework/cluster_rust.go
+359
View File
@@ -0,0 +1,359 @@
package framework
import (
"bytes"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// RustCluster wraps a Go master + Rust volume server for integration testing.
type RustCluster struct {
testingTB testing.TB
profile matrix.Profile
weedBinary string // Go weed binary (for the master)
rustVolumeBinary string // Rust volume binary
baseDir string
configDir string
logsDir string
keepLogs bool
masterPort int
masterGrpcPort int
volumePort int
volumeGrpcPort int
volumePubPort int
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
cleanupOnce sync.Once
}
var (
rustBinaryOnce sync.Once
rustBinaryPath string
rustBinaryErr error
)
// StartRustVolumeCluster starts a Go master + Rust volume server.
func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
t.Helper()
weedBinary, err := FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("resolve weed binary: %v", err)
}
rustBinary, err := FindOrBuildRustBinary()
if err != nil {
t.Fatalf("resolve rust volume binary: %v", err)
}
baseDir, keepLogs, err := newWorkDir()
if err != nil {
t.Fatalf("create temp test directory: %v", err)
}
configDir := filepath.Join(baseDir, "config")
logsDir := filepath.Join(baseDir, "logs")
masterDataDir := filepath.Join(baseDir, "master")
volumeDataDir := filepath.Join(baseDir, "volume")
for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
t.Fatalf("create %s: %v", dir, mkErr)
}
}
if err = writeSecurityConfig(configDir, profile); err != nil {
t.Fatalf("write security config: %v", err)
}
miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
if err != nil {
t.Fatalf("allocate ports: %v", err)
}
masterPort := miniPorts[0]
masterGrpcPort := masterPort + testutil.GrpcPortOffset
rc := &RustCluster{
testingTB: t,
profile: profile,
weedBinary: weedBinary,
rustVolumeBinary: rustBinary,
baseDir: baseDir,
configDir: configDir,
logsDir: logsDir,
keepLogs: keepLogs,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
volumePort: ports[0],
volumeGrpcPort: ports[1],
volumePubPort: ports[0],
}
if profile.SplitPublicPort {
rc.volumePubPort = ports[2]
}
if err = rc.startMaster(masterDataDir); err != nil {
rc.Stop()
t.Fatalf("start master: %v", err)
}
// Reuse the same HTTP readiness helper via an unexported Cluster shim.
helper := &Cluster{logsDir: logsDir}
if err = helper.waitForHTTP(rc.MasterURL() + "/dir/status"); err != nil {
masterLog := helper.tailLog("master.log")
rc.Stop()
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
}
if err = rc.startRustVolume(volumeDataDir); err != nil {
masterLog := helper.tailLog("master.log")
rc.Stop()
t.Fatalf("start rust volume: %v\nmaster log tail:\n%s", err, masterLog)
}
if err = helper.waitForHTTP(rc.VolumeAdminURL() + "/healthz"); err != nil {
volumeLog := helper.tailLog("volume.log")
rc.Stop()
t.Fatalf("wait for rust volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
if err = helper.waitForTCP(rc.VolumeGRPCAddress()); err != nil {
volumeLog := helper.tailLog("volume.log")
rc.Stop()
t.Fatalf("wait for rust volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
}
t.Cleanup(func() {
rc.Stop()
})
return rc
}
// Stop terminates all processes and cleans temporary files.
func (rc *RustCluster) Stop() {
if rc == nil {
return
}
rc.cleanupOnce.Do(func() {
stopProcess(rc.volumeCmd)
stopProcess(rc.masterCmd)
if !rc.keepLogs && !rc.testingTB.Failed() {
_ = os.RemoveAll(rc.baseDir)
} else if rc.baseDir != "" {
rc.testingTB.Logf("rust volume server integration logs kept at %s", rc.baseDir)
}
})
}
func (rc *RustCluster) startMaster(dataDir string) error {
logFile, err := os.Create(filepath.Join(rc.logsDir, "master.log"))
if err != nil {
return err
}
args := []string{
"-config_dir=" + rc.configDir,
"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(rc.masterPort),
"-port.grpc=" + strconv.Itoa(rc.masterGrpcPort),
"-mdir=" + dataDir,
"-peers=none",
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
"-defaultReplication=000",
}
rc.masterCmd = exec.Command(rc.weedBinary, args...)
rc.masterCmd.Dir = rc.baseDir
rc.masterCmd.Stdout = logFile
rc.masterCmd.Stderr = logFile
return rc.masterCmd.Start()
}
func rustVolumeArgs(
profile matrix.Profile,
configDir string,
masterPort int,
volumePort int,
volumeGrpcPort int,
volumePubPort int,
dataDir string,
) []string {
args := []string{
"--port", strconv.Itoa(volumePort),
"--port.grpc", strconv.Itoa(volumeGrpcPort),
"--port.public", strconv.Itoa(volumePubPort),
"--ip", "127.0.0.1",
"--ip.bind", "127.0.0.1",
"--dir", dataDir,
"--max", "16",
"--master", "127.0.0.1:" + strconv.Itoa(masterPort),
"--volume.allowUntrustedRemoteEndpoints",
"--securityFile", filepath.Join(configDir, "security.toml"),
"--readMode", profile.ReadMode,
"--concurrentUploadLimitMB", strconv.Itoa(profile.ConcurrentUploadLimitMB),
"--concurrentDownloadLimitMB", strconv.Itoa(profile.ConcurrentDownloadLimitMB),
"--preStopSeconds", "0",
}
if profile.InflightUploadTimeout > 0 {
args = append(args, "--inflightUploadDataTimeout", profile.InflightUploadTimeout.String())
}
if profile.InflightDownloadTimeout > 0 {
args = append(args, "--inflightDownloadDataTimeout", profile.InflightDownloadTimeout.String())
}
if profile.FileSizeLimitMB > 0 {
args = append(args, "--fileSizeLimitMB", strconv.Itoa(profile.FileSizeLimitMB))
}
return args
}
func (rc *RustCluster) startRustVolume(dataDir string) error {
logFile, err := os.Create(filepath.Join(rc.logsDir, "volume.log"))
if err != nil {
return err
}
args := rustVolumeArgs(
rc.profile,
rc.configDir,
rc.masterPort,
rc.volumePort,
rc.volumeGrpcPort,
rc.volumePubPort,
dataDir,
)
rc.volumeCmd = exec.Command(rc.rustVolumeBinary, args...)
rc.volumeCmd.Dir = rc.baseDir
rc.volumeCmd.Stdout = logFile
rc.volumeCmd.Stderr = logFile
return rc.volumeCmd.Start()
}
// FindOrBuildRustBinary returns an executable Rust volume binary, building one when needed.
func FindOrBuildRustBinary() (string, error) {
if fromEnv := os.Getenv("RUST_VOLUME_BINARY"); fromEnv != "" {
if isExecutableFile(fromEnv) {
return fromEnv, nil
}
return "", fmt.Errorf("RUST_VOLUME_BINARY is set but not executable: %s", fromEnv)
}
rustBinaryOnce.Do(func() {
// Derive the Rust volume crate directory from this source file's location.
rustCrateDir := ""
releaseBin := ""
if _, file, _, ok := runtime.Caller(0); ok {
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
for _, candidate := range []struct {
dir string
bin string
}{
{"seaweed-volume", "weed-volume"},
{"weed-volume", "weed-volume"},
{filepath.Join("rust", "volume_server"), "weed-volume-rs"},
} {
dir := filepath.Join(repoRoot, candidate.dir)
if isDir(dir) && isFile(filepath.Join(dir, "Cargo.toml")) {
rustCrateDir = dir
releaseBin = filepath.Join(dir, "target", "release", candidate.bin)
break
}
}
}
if rustCrateDir == "" {
rustBinaryErr = fmt.Errorf("unable to detect Rust volume crate directory")
return
}
// Always rebuild once per test process so the harness uses current source
// and features. The crate defaults to 5bytes, so a test binary built
// without 5BytesOffset has to turn it off or the Rust server refuses the
// .vif the Go server just wrote.
args := []string{"build", "--release"}
if types.OffsetSize != 5 {
args = append(args, "--no-default-features")
}
cmd := exec.Command("cargo", args...)
cmd.Dir = rustCrateDir
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
if err := cmd.Run(); err != nil {
rustBinaryErr = fmt.Errorf("build rust volume binary: %w\n%s", err, out.String())
return
}
if !isExecutableFile(releaseBin) {
rustBinaryErr = fmt.Errorf("built rust volume binary is not executable: %s", releaseBin)
return
}
rustBinaryPath = releaseBin
})
if rustBinaryErr != nil {
return "", rustBinaryErr
}
return rustBinaryPath, nil
}
func isDir(path string) bool {
info, err := os.Stat(path)
return err == nil && info.IsDir()
}
func isFile(path string) bool {
info, err := os.Stat(path)
return err == nil && info.Mode().IsRegular()
}
// --- accessor methods (mirror Cluster) ---
func (rc *RustCluster) MasterAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.masterPort))
}
func (rc *RustCluster) VolumeAdminAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumePort))
}
func (rc *RustCluster) VolumePublicAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumePubPort))
}
func (rc *RustCluster) VolumeGRPCAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumeGrpcPort))
}
// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
func (rc *RustCluster) VolumeServerAddress() string {
return fmt.Sprintf("%s.%d", rc.VolumeAdminAddress(), rc.volumeGrpcPort)
}
func (rc *RustCluster) MasterURL() string {
return "http://" + rc.MasterAddress()
}
func (rc *RustCluster) VolumeAdminURL() string {
return "http://" + rc.VolumeAdminAddress()
}
func (rc *RustCluster) VolumePublicURL() string {
return "http://" + rc.VolumePublicAddress()
}
func (rc *RustCluster) BaseDir() string {
return rc.baseDir
}
test/volume_server/framework/cluster_rust_test.go
+38
View File
@@ -0,0 +1,38 @@
package framework
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
func TestRustVolumeArgsIncludeReadMode(t *testing.T) {
profile := matrix.P1()
profile.ReadMode = "redirect"
profile.ConcurrentUploadLimitMB = 7
profile.ConcurrentDownloadLimitMB = 9
profile.InflightUploadTimeout = 3 * time.Second
profile.InflightDownloadTimeout = 4 * time.Second
args := rustVolumeArgs(profile, "/tmp/config", 9333, 18080, 28080, 38080, "/tmp/data")
assertArgPair(t, args, "--readMode", "redirect")
assertArgPair(t, args, "--concurrentUploadLimitMB", "7")
assertArgPair(t, args, "--concurrentDownloadLimitMB", "9")
assertArgPair(t, args, "--inflightUploadDataTimeout", "3s")
assertArgPair(t, args, "--inflightDownloadDataTimeout", "4s")
}
func assertArgPair(t *testing.T, args []string, flag string, want string) {
t.Helper()
for i := 0; i+1 < len(args); i += 2 {
if args[i] == flag {
if args[i+1] != want {
t.Fatalf("%s value mismatch: got %q want %q", flag, args[i+1], want)
}
return
}
}
t.Fatalf("missing %s in args: %v", flag, args)
}
test/volume_server/framework/cluster_with_filer.go
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"strconv"
"testing"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
@@ -25,7 +26,7 @@ func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *Cl
baseCluster := StartSingleVolumeCluster(t, profile)
ports, err := allocatePorts(2)
ports, err := testutil.AllocatePorts(2)
if err != nil {
t.Fatalf("allocate filer ports: %v", err)
}
test/volume_server/framework/volume_fixture.go
+85 -1
View File
@@ -5,6 +5,8 @@ import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"testing"
"time"
@@ -14,11 +16,23 @@ import (
func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
t.Helper()
AllocateVolumeWithReplication(t, client, volumeID, collection, "000")
allocateVolume(t, client, volumeID, collection, "", "")
}
func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection, replication string) {
t.Helper()
allocateVolume(t, client, volumeID, collection, replication, "")
}
// AllocateVolumeWithTTL allocates a volume whose needles expire after ttl
// (e.g. "10m").
func AllocateVolumeWithTTL(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, ttl string) {
t.Helper()
allocateVolume(t, client, volumeID, collection, "", ttl)
}
func allocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, replication, ttl string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -30,6 +44,7 @@ func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeS
VolumeId: volumeID,
Collection: collection,
Replication: replication,
Ttl: ttl,
Version: uint32(needle.GetCurrentVersion()),
})
if err != nil {
@@ -53,6 +68,75 @@ func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data
return DoRequest(t, client, req)
}
// CorruptIndexFile appends garbage bytes to a volume's .idx file on disk so
// that CheckIndexFile detects a size mismatch during scrub.
func CorruptIndexFile(t testing.TB, baseDir string, volumeID uint32) {
t.Helper()
idxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.idx", volumeID))
f, err := os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
t.Fatalf("open idx file for corruption: %v", err)
}
defer f.Close()
if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
t.Fatalf("corrupt idx file: %v", err)
}
}
// EnableMaintenanceMode puts the volume server into maintenance mode.
func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_server_pb.VolumeServerClient) {
t.Helper()
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{
Maintenance: true,
Version: stateResp.GetState().GetVersion(),
},
})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
}
}
// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
// so that needle reads fail during a full scrub due to data file size mismatch.
func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
t.Helper()
datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
// Truncate to superblock size only, removing all needle data.
if err := os.Truncate(datPath, 8); err != nil {
t.Fatalf("truncate dat file for corruption: %v", err)
}
}
// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
// that CheckIndexFile detects a size mismatch during EC index scrub.
func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
t.Helper()
ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
t.Fatalf("open ecx file for corruption: %v", err)
}
defer f.Close()
if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
t.Fatalf("corrupt ecx file: %v", err)
}
}
// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
// reads fail during an EC scrub.
func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
t.Helper()
shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
if err := os.Truncate(shardPath, 1); err != nil {
t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
}
}
func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
t.Helper()
test/volume_server/grpc/admin_extra_test.go
+104 -32
View File
@@ -2,11 +2,15 @@ package volume_server_grpc_test
import (
"context"
"io"
"net/http"
"strings"
"testing"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/cluster"
@@ -18,7 +22,7 @@ func TestVolumeNeedleStatusForUploadedFile(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -57,12 +61,57 @@ func TestVolumeNeedleStatusForUploadedFile(t *testing.T) {
}
}
func TestVolumeNeedleStatusIncludesTtlAndLastModified(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(27)
const needleID = uint64(778901)
const cookie = uint32(0xA1B2C3D6)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, needleID, cookie)
client := framework.NewHTTPClient()
uploadReq := mustNewRequest(t, http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid+"?ttl=7d&ts=1700000000")
uploadReq.Body = io.NopCloser(strings.NewReader("needle-status-ttl-payload"))
uploadReq.ContentLength = int64(len("needle-status-ttl-payload"))
uploadReq.Header.Set("Content-Type", "application/octet-stream")
uploadResp := framework.DoRequest(t, client, uploadReq)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: needleID,
})
if err != nil {
t.Fatalf("VolumeNeedleStatus with ttl failed: %v", err)
}
// Go's ReadTTL normalizes via fitTtlCount: 7d → 1w (7 days = 1 week)
if statusResp.GetTtl() != "1w" {
t.Fatalf("ttl mismatch: got %q want %q", statusResp.GetTtl(), "1w")
}
if statusResp.GetLastModified() != 1700000000 {
t.Fatalf("last modified mismatch: got %d want %d", statusResp.GetLastModified(), 1700000000)
}
}
func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -138,7 +187,7 @@ func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -185,7 +234,7 @@ func TestVolumeConfigureInvalidReplication(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -215,7 +264,7 @@ func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -268,22 +317,25 @@ func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// Volume servers only track their masters as ping peers, so drive the
// success path through the master target and rely on the master->volume
// admission for cross-type coverage.
pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
TargetType: cluster.VolumeServerType,
Target: clusterHarness.VolumeServerAddress(),
TargetType: cluster.MasterType,
Target: clusterHarness.MasterAddress(),
})
if err != nil {
t.Fatalf("Ping target volume server failed: %v", err)
t.Fatalf("Ping master from volume server failed: %v", err)
}
if pingResp.GetRemoteTimeNs() == 0 {
t.Fatalf("expected remote timestamp from ping target volume server")
t.Fatalf("expected remote timestamp from ping master")
}
if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
@@ -311,7 +363,7 @@ func TestVolumeServerLeaveIsIdempotent(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -346,47 +398,63 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
// The volume server gates Ping on a known peer list. Unknown target types
// and addresses outside that list are rejected with InvalidArgument
// before any outbound dial is attempted.
_, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
TargetType: "unknown-type",
Target: "127.0.0.1:12345",
})
if err != nil {
t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err)
if err == nil {
t.Fatalf("Ping unknown target type should be rejected by admission")
}
if unknownResp.GetRemoteTimeNs() != 0 {
t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs())
}
if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() {
t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns")
if got := status.Code(err); got != codes.InvalidArgument {
t.Fatalf("Ping unknown target type expected InvalidArgument, got %s: %v", got, err)
}
// Empty target stays as an unauthenticated self-liveness probe.
emptyTargetResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{})
if err != nil {
t.Fatalf("Ping empty target should not return grpc error, got: %v", err)
}
if emptyTargetResp.GetRemoteTimeNs() != 0 {
t.Fatalf("Ping empty target expected remote_time_ns=0, got %d", emptyTargetResp.GetRemoteTimeNs())
}
if emptyTargetResp.GetStopTimeNs() < emptyTargetResp.GetStartTimeNs() {
t.Fatalf("Ping empty target expected stop_time_ns >= start_time_ns")
}
// 127.0.0.1:1 is not in the seed/current master set, so admission rejects
// the call before it can reach the network.
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
TargetType: cluster.MasterType,
Target: "127.0.0.1:1",
})
if err == nil {
t.Fatalf("Ping master target should fail when target is unreachable")
t.Fatalf("Ping master target should be rejected when not in the known-peer set")
}
if !strings.Contains(err.Error(), "ping master") {
t.Fatalf("Ping master unreachable error mismatch: %v", err)
if got := status.Code(err); got != codes.InvalidArgument {
t.Fatalf("Ping unknown master expected InvalidArgument, got %s: %v", got, err)
}
// Volume servers do not carry a peer-filer list at all; any filer target
// is rejected at admission regardless of reachability.
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
TargetType: cluster.FilerType,
Target: "127.0.0.1:1",
})
if err == nil {
t.Fatalf("Ping filer target should fail when target is unreachable")
t.Fatalf("Ping filer target should be rejected by admission")
}
if !strings.Contains(err.Error(), "ping filer") {
t.Fatalf("Ping filer unreachable error mismatch: %v", err)
if got := status.Code(err); got != codes.InvalidArgument {
t.Fatalf("Ping filer expected InvalidArgument, got %s: %v", got, err)
}
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
@@ -406,7 +474,7 @@ func TestPingMasterTargetSuccess(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -433,6 +501,10 @@ func TestPingFilerTargetSuccess(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
// Volume servers do not maintain a peer-filer index, so drive the
// filer-presence success path through the volume server's master peer:
// reaching that master in a filer-bearing cluster is sufficient signal
// that filer joined the cluster without breaking volume-server ping.
clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -441,16 +513,16 @@ func TestPingFilerTargetSuccess(t *testing.T) {
defer cancel()
resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
TargetType: cluster.FilerType,
Target: clusterHarness.FilerServerAddress(),
TargetType: cluster.MasterType,
Target: clusterHarness.MasterAddress(),
})
if err != nil {
t.Fatalf("Ping filer target success path failed: %v", err)
t.Fatalf("Ping master from filer-bearing cluster failed: %v", err)
}
if resp.GetRemoteTimeNs() == 0 {
t.Fatalf("Ping filer target expected non-zero remote time")
t.Fatalf("Ping master expected non-zero remote time")
}
if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
t.Fatalf("Ping master expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
}
}
test/volume_server/grpc/admin_lifecycle_test.go
+14 -20
View File
@@ -19,7 +19,7 @@ func TestVolumeAdminLifecycleRPCs(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -62,7 +62,7 @@ func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -101,7 +101,7 @@ func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -133,7 +133,7 @@ func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -179,12 +179,14 @@ func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
}
}
func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
// Evacuating a server in maintenance mode ends each move by deleting the source
// volume, so VolumeDelete must stay available in that mode (issue #11066).
func TestMaintenanceModeAllowsVolumeDelete(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -194,22 +196,14 @@ func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
framework.EnableMaintenanceMode(t, ctx, client)
if _, err := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
t.Fatalf("VolumeDelete should succeed in maintenance mode, got: %v", err)
}
_, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
_, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err == nil {
t.Fatalf("VolumeDelete should fail when maintenance mode is enabled")
}
if !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("expected maintenance mode error, got: %v", err)
t.Fatalf("VolumeStatus should fail once the volume is deleted")
}
}
test/volume_server/grpc/admin_readonly_collection_test.go
+33 -19
View File
@@ -16,7 +16,7 @@ func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -61,7 +61,7 @@ func TestVolumeMarkReadonlyPersistTrue(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -93,13 +93,17 @@ func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(75)
framework.AllocateVolume(t, grpcClient, volumeID, "")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// non-existent volumes should return "not found"
_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true})
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err)
@@ -110,29 +114,39 @@ func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err)
}
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
// enter maintenance mode
framework.EnableMaintenanceMode(t, ctx, grpcClient)
// marking readonly only restricts the server further and is the first step
// of evacuating it, so maintenance mode lets it through (issue #11066)
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true})
if err != nil {
t.Fatalf("GetState failed: %v", err)
t.Fatalf("VolumeMarkReadonly should succeed in maintenance mode, got: %v", err)
}
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{
Maintenance: true,
Version: stateResp.GetState().GetVersion(),
},
})
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
t.Fatalf("VolumeStatus after readonly in maintenance failed: %v", err)
}
if !statusResp.GetIsReadOnly() {
t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly in maintenance mode")
}
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err)
}
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1})
// reopening a volume for writes is still refused on a read-only server
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
}
// non-existent volume in maintenance mode should still return "not found"
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98773, Persist: true})
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("VolumeMarkReadonly missing-volume in maintenance error mismatch: %v", err)
}
_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98774})
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("VolumeMarkWritable missing-volume in maintenance error mismatch: %v", err)
}
}
func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
@@ -140,7 +154,7 @@ func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/batch_delete_test.go
+17 -14
View File
@@ -21,7 +21,7 @@ func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -64,7 +64,7 @@ func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -122,12 +122,12 @@ func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) {
}
}
func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) {
func TestBatchDeleteMixedStatusesAndMismatchContinuesProcessing(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -191,29 +191,32 @@ func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) {
}
wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1)
stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
mismatchContinueResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
FileIds: []string{wrongCookieB, fidC},
})
if err != nil {
t.Fatalf("BatchDelete mismatch-stop request failed: %v", err)
t.Fatalf("BatchDelete mismatch-continue request failed: %v", err)
}
if len(stopResp.GetResults()) != 1 {
t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults()))
if len(mismatchContinueResp.GetResults()) != 2 {
t.Fatalf("BatchDelete mismatch-continue expected 2 results, got %d", len(mismatchContinueResp.GetResults()))
}
if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus())
if mismatchContinueResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
t.Fatalf("BatchDelete mismatch-continue result[0] expected 400, got %d", mismatchContinueResp.GetResults()[0].GetStatus())
}
if mismatchContinueResp.GetResults()[1].GetStatus() != http.StatusAccepted {
t.Fatalf("BatchDelete mismatch-continue result[1] expected 202, got %d", mismatchContinueResp.GetResults()[1].GetStatus())
}
readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
_ = framework.ReadAllAndClose(t, readB)
if readB.StatusCode != http.StatusOK {
t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode)
t.Fatalf("fidB should remain after cookie mismatch, got %d", readB.StatusCode)
}
readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC)
_ = framework.ReadAllAndClose(t, readC)
if readC.StatusCode != http.StatusOK {
t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode)
if readC.StatusCode != http.StatusNotFound {
t.Fatalf("fidC should be deleted when processing continues past a mismatch, got %d", readC.StatusCode)
}
}
@@ -222,7 +225,7 @@ func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/copy_receive_variants_test.go
+69 -10
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"math"
"net/http"
"strings"
"testing"
"time"
@@ -18,7 +19,7 @@ func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -77,7 +78,7 @@ func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -89,7 +90,7 @@ func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".definitely-missing",
Ext: ".missing",
CompactionRevision: math.MaxUint32,
StopOffset: 1,
IgnoreSourceFileNotFound: false,
@@ -103,7 +104,7 @@ func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".definitely-missing",
Ext: ".missing",
CompactionRevision: math.MaxUint32,
StopOffset: 1,
IgnoreSourceFileNotFound: true,
@@ -118,7 +119,7 @@ func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".definitely-missing",
Ext: ".missing",
CompactionRevision: math.MaxUint32,
StopOffset: 0,
IgnoreSourceFileNotFound: false,
@@ -132,12 +133,70 @@ func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
}
}
func TestCopyFileStopOffsetZeroExistingFileSendsMetadata(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(93)
const needleID = uint64(770101)
const cookie = uint32(0x1234ABCD)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
uploadResp := framework.UploadBytes(
t,
client,
clusterHarness.VolumeAdminURL(),
framework.NewFileID(volumeID, needleID, cookie),
[]byte("copy-file-stop-zero"),
)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".dat",
CompactionRevision: math.MaxUint32,
StopOffset: 0,
IgnoreSourceFileNotFound: false,
})
if err != nil {
t.Fatalf("CopyFile stop_offset=0 existing file start failed: %v", err)
}
msg, err := stream.Recv()
if err != nil {
t.Fatalf("CopyFile stop_offset=0 existing file recv failed: %v", err)
}
if len(msg.GetFileContent()) != 0 {
t.Fatalf("CopyFile stop_offset=0 existing file should not send content, got %d bytes", len(msg.GetFileContent()))
}
if msg.GetModifiedTsNs() == 0 {
t.Fatalf("CopyFile stop_offset=0 existing file expected non-zero ModifiedTsNs")
}
_, err = stream.Recv()
if err != io.EOF {
t.Fatalf("CopyFile stop_offset=0 existing file expected EOF after metadata frame, got: %v", err)
}
}
func TestCopyFileCompactionRevisionMismatch(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -166,7 +225,7 @@ func TestReceiveFileProtocolViolationResponses(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -213,7 +272,7 @@ func TestReceiveFileSuccessForRegularVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -299,7 +358,7 @@ func TestReceiveFileSuccessForEcVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -389,7 +448,7 @@ func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/copy_sync_test.go
+114 -3
View File
@@ -3,6 +3,7 @@ package volume_server_grpc_test
import (
"context"
"io"
"math/rand"
"net/http"
"strings"
"testing"
@@ -18,12 +19,19 @@ func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
httpClient := framework.NewHTTPClient()
const volumeID = uint32(41)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 1, 0x11112222)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("sync-status-payload"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -46,6 +54,12 @@ func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) {
if statusResp.GetVersion() == 0 {
t.Fatalf("ReadVolumeFileStatus expected non-zero version")
}
if syncResp.GetTailOffset() == 0 {
t.Fatalf("VolumeSyncStatus expected non-zero tail offset after upload")
}
if syncResp.GetTailOffset() != statusResp.GetDatFileSize() {
t.Fatalf("VolumeSyncStatus tail offset mismatch: got %d want %d", syncResp.GetTailOffset(), statusResp.GetDatFileSize())
}
}
func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) {
@@ -53,7 +67,7 @@ func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -100,7 +114,7 @@ func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -282,3 +296,100 @@ func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) {
t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload))
}
}
// A VolumeCopy whose caller goes away must not leave a mounted volume behind on
// the destination. weed-admin's batch balance routinely starts far more copies
// than it finishes, and each abandoned copy that still mounts costs the
// destination a volume it was never asked to hold: a per-volume index cache that
// is never reclaimed, and — under replication=000 — a second writable copy of a
// volume id that two writers can diverge.
//
// The copy is throttled so it is demonstrably still in flight when the caller
// cancels; the assertion before the cancel keeps the test from passing
// vacuously if the throttle ever stops biting.
func TestVolumeCopyCancelledByCallerDoesNotMountDestination(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
defer sourceConn.Close()
destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
defer destConn.Close()
const volumeID = uint32(44)
const payloadMiB = 192
framework.AllocateVolume(t, sourceClient, volumeID, "")
// The fixture size is load-bearing twice over; do not shrink it to save CI
// time without reading both reasons.
//
// First, the copy has to still be running when the caller cancels, and the
// only lever for that is IoBytePerSecond. It is a no-op below ~100ms of wall
// clock — the throttler never re-checks within its first window — and on a
// tmpfs loopback cluster 64 MiB copies in ~110ms.
//
// Second, 192 MiB is above the 128 MiB progress-report interval, which is
// what lets this test be green on Go: a failing stream.Send from that report
// is Go's only abort signal. Measured below the interval (120 MiB), a Go
// destination mounts the abandoned copy too, so a smaller fixture would fail
// the Go leg without any Rust change.
//
// The bytes are pseudo-random because the volume server gzips compressible
// uploads; a repeating payload lands on disk as a few KB and the throttle
// never bites.
httpClient := framework.NewHTTPClient()
chunk := make([]byte, 1024*1024)
if _, err := rand.New(rand.NewSource(11186)).Read(chunk); err != nil {
t.Fatalf("generate payload: %v", err)
}
for i := 0; i < payloadMiB; i++ {
fid := framework.NewFileID(volumeID, uint64(880100+i), 0x3456789a)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, chunk)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload %d to source expected 201, got %d", i, uploadResp.StatusCode)
}
}
// 192 MiB at 16 MiB/s is about twelve seconds of copying.
copyCtx, cancelCopy := context.WithCancel(context.Background())
defer cancelCopy()
_, err := destClient.VolumeCopy(copyCtx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
Collection: "",
SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
IoBytePerSecond: 16 * 1024 * 1024,
})
if err != nil {
t.Fatalf("VolumeCopy start failed: %v", err)
}
time.Sleep(2 * time.Second)
// Premise: the copy must still be running when we cancel. If the
// destination has already mounted, the throttle no longer bites and the
// rest of this test would prove nothing.
premiseCtx, cancelPremise := context.WithTimeout(context.Background(), 5*time.Second)
_, premiseErr := destClient.ReadVolumeFileStatus(premiseCtx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
cancelPremise()
if premiseErr == nil {
t.Fatalf("copy of volume %d already completed before the cancel; raise payloadMiB or lower IoBytePerSecond", volumeID)
}
cancelCopy()
// Well past the ~10s the unabandoned copy would have needed: the volume
// must never appear on the destination.
deadline := time.Now().Add(25 * time.Second)
for time.Now().Before(deadline) {
pollCtx, cancelPoll := context.WithTimeout(context.Background(), 5*time.Second)
_, statusErr := destClient.ReadVolumeFileStatus(pollCtx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
cancelPoll()
if statusErr == nil {
t.Fatalf("destination mounted volume %d after its VolumeCopy caller cancelled", volumeID)
}
time.Sleep(500 * time.Millisecond)
}
}
test/volume_server/grpc/data_rw_test.go
+3 -3
View File
@@ -17,7 +17,7 @@ func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -55,7 +55,7 @@ func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -105,7 +105,7 @@ func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/data_stream_success_test.go
+199 -46
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"net/http"
"reflect"
"strings"
"testing"
"time"
@@ -20,7 +21,7 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -123,7 +124,7 @@ func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -181,7 +182,7 @@ func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -231,12 +232,206 @@ func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
}
}
func TestReadAllNeedlesPreservesDatOrderAcrossOverwrite(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(86)
const firstNeedleID = uint64(444551)
const secondNeedleID = uint64(444552)
const firstCookie = uint32(0xAA22BB33)
const secondCookie = uint32(0xCC44DD55)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
uploads := []struct {
fid string
body string
}{
{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first"},
{fid: framework.NewFileID(volumeID, secondNeedleID, secondCookie), body: "read-all-second"},
{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first-overwrite"},
}
for _, upload := range uploads {
resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), upload.fid, []byte(upload.body))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != 201 {
t.Fatalf("upload for %s expected 201, got %d", upload.fid, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
t.Fatalf("ReadAllNeedles start failed: %v", err)
}
var orderedIDs []uint64
var orderedBodies []string
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
}
orderedIDs = append(orderedIDs, msg.GetNeedleId())
orderedBodies = append(orderedBodies, string(msg.GetNeedleBlob()))
}
wantIDs := []uint64{secondNeedleID, firstNeedleID}
wantBodies := []string{"read-all-second", "read-all-first-overwrite"}
if !reflect.DeepEqual(orderedIDs, wantIDs) {
t.Fatalf("ReadAllNeedles order mismatch: got %v want %v", orderedIDs, wantIDs)
}
if !reflect.DeepEqual(orderedBodies, wantBodies) {
t.Fatalf("ReadAllNeedles bodies mismatch: got %v want %v", orderedBodies, wantBodies)
}
}
func TestReadNeedleMetaDeletedEntryUsesTombstoneMetadata(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(87)
const needleID = uint64(444661)
const cookie = uint32(0xAB12CD34)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, needleID, cookie)
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-meta-delete"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid, nil)
if err != nil {
t.Fatalf("build delete request: %v", err)
}
deleteResp := framework.DoRequest(t, client, deleteReq)
_ = framework.ReadAllAndClose(t, deleteResp)
if deleteResp.StatusCode != http.StatusAccepted {
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("ReadVolumeFileStatus after delete failed: %v", err)
}
idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".idx",
CompactionRevision: fileStatus.GetCompactionRevision(),
StopOffset: fileStatus.GetIdxFileSize(),
})
offset, size := findLastNeedleOffsetAndSize(t, idxBytes, needleID)
if size >= 0 {
t.Fatalf("expected deleted idx entry for needle %d, got size %d", needleID, size)
}
metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
VolumeId: volumeID,
NeedleId: needleID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleMeta deleted-entry failed: %v", err)
}
if metaResp.GetCookie() != cookie {
t.Fatalf("ReadNeedleMeta deleted-entry cookie mismatch: got %d want %d", metaResp.GetCookie(), cookie)
}
if metaResp.GetAppendAtNs() == 0 {
t.Fatalf("ReadNeedleMeta deleted-entry expected non-zero append_at_ns")
}
}
func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.CopyFile(ctx, req)
if err != nil {
t.Fatalf("CopyFile start failed: %v", err)
}
var out []byte
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
return out
}
if recvErr != nil {
t.Fatalf("CopyFile recv failed: %v", recvErr)
}
out = append(out, msg.GetFileContent()...)
}
}
func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
if entryOffset.IsZero() || entrySize <= 0 {
continue
}
return entryOffset.ToActualOffset(), int32(entrySize)
}
t.Fatalf("needle id %d not found in idx entries", needleID)
return 0, 0
}
func findLastNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
found := false
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
found = true
offset = entryOffset.ToActualOffset()
size = int32(entrySize)
}
if !found {
t.Fatalf("needle id %d not found in idx entries", needleID)
}
return offset, size
}
func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -295,45 +490,3 @@ func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume B needle: got %q want %q", got, payloadB)
}
}
func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.CopyFile(ctx, req)
if err != nil {
t.Fatalf("CopyFile start failed: %v", err)
}
var out []byte
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
return out
}
if recvErr != nil {
t.Fatalf("CopyFile recv failed: %v", recvErr)
}
out = append(out, msg.GetFileContent()...)
}
}
func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
if entryOffset.IsZero() || entrySize <= 0 {
continue
}
return entryOffset.ToActualOffset(), int32(entrySize)
}
t.Fatalf("needle id %d not found in idx entries", needleID)
return 0, 0
}
test/volume_server/grpc/ec_decode_ecj_test.go
+294
View File
@@ -0,0 +1,294 @@
package volume_server_grpc_test
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)
// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
// decoded volume produced by VolumeEcShardsToVolume.
func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(140)
keyA = uint64(990020)
cookieA = uint32(0xDA001122)
keyB = uint64(990021)
cookieB = uint32(0xDA003344)
)
framework.AllocateVolume(t, client, volumeID, "")
httpClient := framework.NewHTTPClient()
fidA := framework.NewFileID(volumeID, keyA, cookieA)
fidB := framework.NewFileID(volumeID, keyB, cookieB)
payloadA := []byte("needle-A-should-be-deleted-after-decode")
payloadB := []byte("needle-B-should-survive-decode")
// Upload two needles.
for _, tc := range []struct {
fid string
payload []byte
}{
{fidA, payloadA},
{fidB, payloadB},
} {
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// EC encode.
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate: %v", err)
}
// Mount all data shards so the EC volume is usable.
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount: %v", err)
}
// Delete needle A via EC path (writes to .ecj).
_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
VolumeId: volumeID, Collection: "",
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
})
if err != nil {
t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
}
// Unmount the normal volume so decode writes fresh files.
_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeUnmount: %v", err)
}
// Decode EC shards back to a normal volume.
_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsToVolume: %v", err)
}
// Re-mount the decoded volume.
_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeMount: %v", err)
}
// Needle A should be gone (deleted via .ecj before decode).
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
bodyA := framework.ReadAllAndClose(t, respA)
if respA.StatusCode >= 500 {
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
}
if respA.StatusCode != http.StatusNotFound {
t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
}
// Needle B should still be readable.
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
bodyB := framework.ReadAllAndClose(t, respB)
if respB.StatusCode != http.StatusOK {
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
}
if string(bodyB) != string(payloadB) {
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
}
}
// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
// peer server that contributes no new data shards are still collected during
// decode. This is the regression test for the fix in collectEcShards that
// always copies .ecj from every shard location.
//
// Scenario:
// - Server 0 holds all 10 data shards (decode target).
// - Server 1 holds a copy of shard 0 (no new shards for server 0).
// - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
// - During decode on server 0, server 1's .ecj must be collected and applied.
func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const (
volumeID = uint32(141)
keyA = uint64(990030)
cookieA = uint32(0xDB001122)
keyB = uint64(990031)
cookieB = uint32(0xDB003344)
)
// Allocate and upload on server 0.
framework.AllocateVolume(t, client0, volumeID, "")
httpClient := framework.NewHTTPClient()
fidA := framework.NewFileID(volumeID, keyA, cookieA)
fidB := framework.NewFileID(volumeID, keyB, cookieB)
payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
}
resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// EC encode on server 0.
_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
}
// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
// Copy shard 0 + ecx + ecj from server 0 → server 1.
_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: volumeID,
Collection: "",
SourceDataNode: sourceDataNode,
ShardIds: []uint32{0},
CopyEcxFile: true,
CopyEcjFile: true,
CopyVifFile: true,
})
if err != nil {
t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
}
// Mount shard 0 on server 1 so the EC volume can accept deletions.
_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
}
// Delete needle A on server 1 only (creates .ecj entry on server 1).
_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
VolumeId: volumeID, Collection: "",
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
})
if err != nil {
t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
}
// Mount all data shards on server 0 (the decode target).
_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
}
// Collect .ecj from server 1 → server 0 with NO new shard IDs.
// This is the critical path: server 1 has shard 0 which server 0 already
// has, so needToCopyShardsInfo would be empty. Before the fix in
// collectEcShards, this copy would be skipped entirely, losing server 1's
// deletion entries.
server1DataNode := cluster.VolumeAdminAddress(1) + "." +
strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: volumeID,
Collection: "",
SourceDataNode: server1DataNode,
ShardIds: []uint32{}, // No new shards — just .ecj.
CopyEcxFile: false,
CopyEcjFile: true,
CopyVifFile: false,
})
if err != nil {
t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
}
// Unmount the normal volume before decode.
_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeUnmount on server 0: %v", err)
}
// Decode on server 0. RebuildEcxFile should see needle A's deletion from
// the .ecj that was collected from server 1.
_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
}
// Re-mount the decoded normal volume.
_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeMount on server 0: %v", err)
}
// Needle A should be gone — its deletion was in server 1's .ecj.
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
bodyA := framework.ReadAllAndClose(t, respA)
if respA.StatusCode >= 500 {
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
}
if respA.StatusCode != http.StatusNotFound {
t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
}
// Needle B should still be readable.
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
bodyB := framework.ReadAllAndClose(t, respB)
if respB.StatusCode != http.StatusOK {
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
}
if string(bodyB) != string(payloadB) {
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
}
}
test/volume_server/grpc/ec_multi_disk_lifecycle_test.go
+529
View File
@@ -0,0 +1,529 @@
package volume_server_grpc_test
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"sort"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)
// TestEcLifecycleAcrossMultipleDisks drives encode, mount, read, drop-dat,
// cross-disk redistribute + restart + reconcile, blob delete, and rebuild on
// a 2-disk volume server.
func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const (
dataDirCount = 2
volumeID = uint32(9500)
collection = "ec-multi-disk"
)
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
dataDirs := clusterHarness.VolumeDataDirs()
if len(dataDirs) != dataDirCount {
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
}
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
framework.AllocateVolume(t, grpcClient, volumeID, collection)
// populate
type needleSpec struct {
fid string
payload []byte
}
needles := []needleSpec{
{framework.NewFileID(volumeID, 9501, 0xAA000001), bytesOfLen(11, 0xA1)},
{framework.NewFileID(volumeID, 9502, 0xAA000002), bytesOfLen(1024, 0xA2)},
{framework.NewFileID(volumeID, 9503, 0xAA000003), bytesOfLen(63*1024+17, 0xA3)},
{framework.NewFileID(volumeID, 9504, 0xAA000004), bytesOfLen(2, 0xA4)},
{framework.NewFileID(volumeID, 9505, 0xAA000005), bytesOfLen(513*1024, 0xA5)},
}
httpClient := framework.NewHTTPClient()
for _, n := range needles {
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
}
}
// encode — every shard lands on the .dat's disk
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: collection,
}); err != nil {
t.Fatalf("VolumeEcShardsGenerate: %v", err)
}
postGenerateLayout := scanShardLayout(t, dataDirs, collection, volumeID)
if len(postGenerateLayout[0]) != erasure_coding.TotalShardsCount {
t.Fatalf("post-generate: expected all %d shards on disk 0, got per-disk layout %v",
erasure_coding.TotalShardsCount, postGenerateLayout)
}
if len(postGenerateLayout[1]) != 0 {
t.Fatalf("post-generate: disk 1 should be empty, got %v", postGenerateLayout[1])
}
// .ecj is created lazily on first EC delete
for _, side := range []string{".ecx", ".vif"} {
if !fileExistsIn(dataDirs[0], collection, volumeID, side) {
t.Fatalf("post-generate: expected %s on disk 0", side)
}
}
// mount + read via EC
if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: collection,
ShardIds: dataShardIds(),
}); err != nil {
t.Fatalf("VolumeEcShardsMount data shards: %v", err)
}
for _, n := range needles {
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-mount")
}
// drop the original volume so EC shards are the only source
if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
VolumeId: volumeID,
}); err != nil {
t.Fatalf("VolumeDelete (drop .dat): %v", err)
}
if fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
t.Fatalf("VolumeDelete left .dat in place on disk 0")
}
for _, n := range needles {
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-dat-delete")
}
// move half the shards onto disk 1, leaving .ecx on disk 0
clusterHarness.StopVolumeServer()
const splitAt = 7
for shard := 0; shard < splitAt; shard++ {
movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
}
postMoveLayout := scanShardLayout(t, dataDirs, collection, volumeID)
if len(postMoveLayout[0]) != erasure_coding.TotalShardsCount-splitAt {
t.Fatalf("post-move: expected %d shards on disk 0, got %v", erasure_coding.TotalShardsCount-splitAt, postMoveLayout[0])
}
if len(postMoveLayout[1]) != splitAt {
t.Fatalf("post-move: expected %d shards on disk 1, got %v", splitAt, postMoveLayout[1])
}
if !fileExistsIn(dataDirs[0], collection, volumeID, ".ecx") {
t.Fatalf("post-move: .ecx must stay on disk 0 to exercise the cross-disk reconcile path")
}
if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
t.Fatalf("post-move: .ecx leaked onto disk 1; reconcile path would not be exercised")
}
clusterHarness.RestartVolumeServer()
conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn2.Close()
postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
}
if got, want := len(postReconcileLayout[0]), erasure_coding.TotalShardsCount-splitAt; got != want {
t.Fatalf("post-reconcile: disk 0 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
}
if got, want := len(postReconcileLayout[1]), splitAt; got != want {
t.Fatalf("post-reconcile: disk 1 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
}
// VolumeEcShardsInfo must walk every DiskLocation and report the full
// shard set — the verification step in ec_task.go gates source-volume
// deletion on this RPC returning a complete shard inventory.
infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("VolumeEcShardsInfo after redistribute restart: %v", err)
}
if got, want := len(infoResp.GetEcShardInfos()), erasure_coding.TotalShardsCount; got != want {
t.Fatalf("VolumeEcShardsInfo after redistribute restart: got %d shards, want %d (per-disk layout=%v)",
got, want, postReconcileLayout)
}
gotShardIds := make(map[uint32]struct{}, len(infoResp.GetEcShardInfos()))
for _, info := range infoResp.GetEcShardInfos() {
gotShardIds[info.GetShardId()] = struct{}{}
}
for shardId := uint32(0); shardId < uint32(erasure_coding.TotalShardsCount); shardId++ {
if _, ok := gotShardIds[shardId]; !ok {
t.Fatalf("VolumeEcShardsInfo missing shard %d (per-disk layout=%v)", shardId, postReconcileLayout)
}
}
for _, n := range needles {
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-cross-disk-reconcile")
}
// blob delete on a cross-disk layout
deletedNeedle := needles[2]
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+deletedNeedle.fid, nil)
if err != nil {
t.Fatalf("build delete request: %v", err)
}
deleteResp := framework.DoRequest(t, httpClient, deleteReq)
_ = framework.ReadAllAndClose(t, deleteResp)
if deleteResp.StatusCode != http.StatusAccepted {
t.Fatalf("HTTP DELETE %s expected 202, got %d", deletedNeedle.fid, deleteResp.StatusCode)
}
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), deletedNeedle.fid)
_ = framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode == http.StatusOK {
t.Fatalf("HTTP GET on deleted EC needle expected non-200, got %d", readResp.StatusCode)
}
for _, n := range needles {
if n.fid == deletedNeedle.fid {
continue
}
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-blob-delete")
}
// rebuild a shard whose inputs are split across two disks
clusterHarness.StopVolumeServer()
const repairTargetShard = 9 // sits on disk 0 after the split
shardPath := filepath.Join(dataDirs[0], shardFileName(collection, volumeID, repairTargetShard))
if _, statErr := os.Stat(shardPath); statErr != nil {
t.Fatalf("repair setup: shard %d not on disk 0 (%s): %v", repairTargetShard, shardPath, statErr)
}
if err := os.Remove(shardPath); err != nil {
t.Fatalf("repair setup: remove shard %d on disk 0: %v", repairTargetShard, err)
}
clusterHarness.RestartVolumeServer()
conn3, grpcClient3 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn3.Close()
rebuildResp, err := grpcClient3.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
VolumeId: volumeID,
Collection: collection,
})
if err != nil {
t.Fatalf("VolumeEcShardsRebuild after cross-disk shard loss: %v", err)
}
if !containsUint32(rebuildResp.GetRebuiltShardIds(), repairTargetShard) {
t.Fatalf("VolumeEcShardsRebuild expected to rebuild shard %d, got %v",
repairTargetShard, rebuildResp.GetRebuiltShardIds())
}
// The rebuilder picks whichever disk hosts the most shards plus a
// matching .ecx. After the boot-time mirror runs, every shard-
// bearing disk owns its own sidecars, so rebuild can legitimately
// restore the shard on either disk — accept both.
rebuiltOn := -1
for i, dir := range dataDirs {
candidate := filepath.Join(dir, shardFileName(collection, volumeID, repairTargetShard))
if _, statErr := os.Stat(candidate); statErr == nil {
rebuiltOn = i
shardPath = candidate
break
}
}
if rebuiltOn < 0 {
t.Fatalf("rebuild did not restore shard %d on any disk (checked %v)", repairTargetShard, dataDirs)
}
if _, err := grpcClient3.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: collection,
ShardIds: []uint32{repairTargetShard},
}); err != nil {
t.Fatalf("VolumeEcShardsMount rebuilt shard %d: %v", repairTargetShard, err)
}
for _, n := range needles {
if n.fid == deletedNeedle.fid {
continue
}
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-repair")
}
}
// TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart seeds a healthy .dat on
// disk 0, plants the on-disk footprint of an interrupted EC encode on disk 1
// (one shard plus .ecx / .ecj / .vif, no .dat), and asserts the startup prune
// wipes disk 1 without touching disk 0.
func TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const (
dataDirCount = 2
volumeID = uint32(9501)
collection = "ec-9478"
partialShard = 1
)
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
dataDirs := clusterHarness.VolumeDataDirs()
if len(dataDirs) != dataDirCount {
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
}
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
needleFid := framework.NewFileID(volumeID, 7777, 0xCAFEBABE)
needlePayload := bytesOfLen(4096, 0xC7)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload)
_ = framework.ReadAllAndClose(t, upResp)
if upResp.StatusCode != http.StatusCreated {
t.Fatalf("seed upload expected 201, got %d", upResp.StatusCode)
}
datPath := filepath.Join(dataDirs[0], datFileName(collection, volumeID))
datInfo, err := os.Stat(datPath)
if err != nil {
t.Fatalf("seed setup: .dat must exist on disk 0 (%s): %v", datPath, err)
}
// prune's size guard refuses cleanup if the sibling .dat looks truncated
if datInfo.Size() == 0 {
t.Fatalf("seed setup: .dat on disk 0 is zero bytes")
}
clusterHarness.StopVolumeServer()
plantPartialEc(t, dataDirs[1], collection, volumeID, partialShard, datInfo.Size())
preRestartLayout := scanShardLayout(t, dataDirs, collection, volumeID)
if len(preRestartLayout[0]) != 0 {
t.Fatalf("seed setup: disk 0 should hold no EC shards before restart, got %v", preRestartLayout[0])
}
if len(preRestartLayout[1]) != 1 {
t.Fatalf("seed setup: disk 1 should hold exactly one planted shard, got %v", preRestartLayout[1])
}
clusterHarness.RestartVolumeServer()
conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn2.Close()
// the volume admin port answers before the startup prune finishes
postPruneLayout := waitForLayout(t, dataDirs, collection, volumeID, func(layout map[int][]int) bool {
return len(layout[1]) == 0
}, 10*time.Second)
if len(postPruneLayout[1]) != 0 {
t.Fatalf("partial EC shards survived restart on disk 1: %v", postPruneLayout[1])
}
// .vif is intentionally left behind: with no .ecx / shard to anchor it,
// the next startup ignores it.
for _, side := range []string{".ecx", ".ecj"} {
if fileExistsIn(dataDirs[1], collection, volumeID, side) {
t.Fatalf("%s survived prune on disk 1", side)
}
}
if !fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
t.Fatalf("prune wiped the healthy .dat on disk 0")
}
if !fileExistsIn(dataDirs[0], collection, volumeID, ".idx") {
t.Fatalf("prune wiped the healthy .idx on disk 0")
}
if _, err = grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
VolumeId: volumeID,
}); err == nil {
t.Fatalf("VolumeEcShardsInfo returned success after prune; the partial EC mount should be gone")
}
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload, "after-prune")
}
func bytesOfLen(n int, seed byte) []byte {
out := make([]byte, n)
for i := range out {
out[i] = byte(int(seed) + i*31)
}
return out
}
func dataShardIds() []uint32 {
ids := make([]uint32, erasure_coding.DataShardsCount)
for i := range ids {
ids[i] = uint32(i)
}
return ids
}
func datFileName(collection string, volumeID uint32) string {
if collection == "" {
return fmt.Sprintf("%d.dat", volumeID)
}
return fmt.Sprintf("%s_%d.dat", collection, volumeID)
}
func shardFileName(collection string, volumeID uint32, shardID int) string {
if collection == "" {
return fmt.Sprintf("%d%s", volumeID, erasure_coding.ToExt(shardID))
}
return fmt.Sprintf("%s_%d%s", collection, volumeID, erasure_coding.ToExt(shardID))
}
func sideFileName(collection string, volumeID uint32, ext string) string {
if collection == "" {
return fmt.Sprintf("%d%s", volumeID, ext)
}
return fmt.Sprintf("%s_%d%s", collection, volumeID, ext)
}
func fileExistsIn(dir, collection string, volumeID uint32, ext string) bool {
_, err := os.Stat(filepath.Join(dir, sideFileName(collection, volumeID, ext)))
return err == nil
}
func scanShardLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32) map[int][]int {
t.Helper()
out := make(map[int][]int, len(dataDirs))
for diskIdx, dir := range dataDirs {
out[diskIdx] = nil
for shard := 0; shard < erasure_coding.TotalShardsCount; shard++ {
if _, err := os.Stat(filepath.Join(dir, shardFileName(collection, volumeID, shard))); err == nil {
out[diskIdx] = append(out[diskIdx], shard)
}
}
sort.Ints(out[diskIdx])
}
return out
}
func waitForLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32, pred func(map[int][]int) bool, timeout time.Duration) map[int][]int {
t.Helper()
deadline := time.Now().Add(timeout)
var layout map[int][]int
for time.Now().Before(deadline) {
layout = scanShardLayout(t, dataDirs, collection, volumeID)
if pred(layout) {
return layout
}
time.Sleep(100 * time.Millisecond)
}
return layout
}
func movedFile(t testing.TB, fromDir, toDir, collection string, volumeID uint32, ext string) {
t.Helper()
name := sideFileName(collection, volumeID, ext)
src := filepath.Join(fromDir, name)
dst := filepath.Join(toDir, name)
if err := os.Rename(src, dst); err != nil {
t.Fatalf("move %s → %s: %v", src, dst, err)
}
}
func totalShardsInLayout(layout map[int][]int) int {
n := 0
for _, ids := range layout {
n += len(ids)
}
return n
}
func containsUint32(s []uint32, v uint32) bool {
for _, x := range s {
if x == v {
return true
}
}
return false
}
func verifyHTTPRead(t testing.TB, client *http.Client, base, fid string, want []byte, stage string) {
t.Helper()
resp := framework.ReadBytes(t, client, base, fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("[%s] GET %s expected 200, got %d", stage, fid, resp.StatusCode)
}
if len(body) != len(want) {
t.Fatalf("[%s] GET %s size mismatch: got %d, want %d", stage, fid, len(body), len(want))
}
for i := range want {
if body[i] != want[i] {
t.Fatalf("[%s] GET %s byte %d mismatch: got %d, want %d", stage, fid, i, body[i], want[i])
}
}
}
// plantPartialEc writes the footprint of an interrupted EC encode (one shard
// sized for datFileSize, plus .ecx / .ecj / .vif, no .dat) into dir.
func plantPartialEc(t testing.TB, dir, collection string, volumeID uint32, shardID int, datFileSize int64) {
t.Helper()
shardPath := filepath.Join(dir, shardFileName(collection, volumeID, shardID))
if f, err := os.Create(shardPath); err == nil {
expectedShardSize := plantedShardSize(datFileSize)
if err := f.Truncate(expectedShardSize); err != nil {
f.Close()
t.Fatalf("truncate planted shard: %v", err)
}
f.Close()
} else {
t.Fatalf("create planted shard %s: %v", shardPath, err)
}
for _, side := range []struct {
ext string
content []byte
}{
{".ecx", []byte("dummy ecx")},
{".ecj", nil},
} {
p := filepath.Join(dir, sideFileName(collection, volumeID, side.ext))
f, err := os.Create(p)
if err != nil {
t.Fatalf("create planted %s: %v", side.ext, err)
}
if side.content != nil {
if _, err := f.Write(side.content); err != nil {
f.Close()
t.Fatalf("write planted %s: %v", side.ext, err)
}
}
f.Close()
}
// Record the encode-time source size so the prune's byte-exact gate
// recognizes the sibling .dat as the source, as a real encoded volume does.
vifPath := filepath.Join(dir, sideFileName(collection, volumeID, ".vif"))
if err := volume_info.SaveVolumeInfo(vifPath, &volume_server_pb.VolumeInfo{
Version: uint32(needle.Version3),
DatFileSize: datFileSize,
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: uint32(erasure_coding.DataShardsCount),
ParityShards: uint32(erasure_coding.ParityShardsCount),
},
}); err != nil {
t.Fatalf("save planted .vif: %v", err)
}
}
// mirrors calculateExpectedShardSize in weed/storage/disk_location_ec.go
func plantedShardSize(datFileSize int64) int64 {
const dataShards = int64(erasure_coding.DataShardsCount)
largeBatch := int64(erasure_coding.ErasureCodingLargeBlockSize) * dataShards
numLarge := datFileSize / largeBatch
shardSize := numLarge * int64(erasure_coding.ErasureCodingLargeBlockSize)
remaining := datFileSize - numLarge*largeBatch
if remaining > 0 {
smallBatch := int64(erasure_coding.ErasureCodingSmallBlockSize) * dataShards
numSmall := (remaining + smallBatch - 1) / smallBatch
shardSize += numSmall * int64(erasure_coding.ErasureCodingSmallBlockSize)
}
return shardSize
}
test/volume_server/grpc/ec_receive_disk_placement_test.go
+178
View File
@@ -0,0 +1,178 @@
package volume_server_grpc_test
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// Sends EC shards with disk_id={1, 2, 0} and verifies each lands on the
// requested disk (0 → auto-select → disk 0 for a fresh volume).
func TestReceiveFileEcShardHonorsDiskID(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const dataDirCount = 3
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
dataDirs := clusterHarness.VolumeDataDirs()
if len(dataDirs) != dataDirCount {
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
const volumeID = uint32(9184)
const collection = "ec-disk-placement"
// DiskId=0 triggers auto-select. Since this test never calls
// VolumeEcShardsMount, loc.FindEcVolume returns false on every disk, so
// the "already holds shards" preference is skipped and auto-select falls
// through to the first HDD (dir 0). If shards get mounted between uploads
// in a future revision, the expected dir for DiskId=0 becomes the first
// disk that already holds a shard.
shards := []struct {
shardID uint32
requestedDisk uint32
expectedDirIdx int
}{
{shardID: 0, requestedDisk: 1, expectedDirIdx: 1},
{shardID: 5, requestedDisk: 2, expectedDirIdx: 2},
{shardID: 13, requestedDisk: 0, expectedDirIdx: 0},
}
shardExt := func(id uint32) string { return fmt.Sprintf(".ec%02d", id) }
for _, s := range shards {
payload := []byte(fmt.Sprintf("ec-shard-payload-%02d", s.shardID))
stream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create for shard %d: %v", s.shardID, err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: shardExt(s.shardID),
Collection: collection,
IsEcVolume: true,
ShardId: s.shardID,
FileSize: uint64(len(payload)),
DiskId: s.requestedDisk,
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info for shard %d: %v", s.shardID, err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payload},
}); err != nil {
t.Fatalf("ReceiveFile send content for shard %d: %v", s.shardID, err)
}
resp, err := stream.CloseAndRecv()
if err != nil {
t.Fatalf("ReceiveFile close for shard %d: %v", s.shardID, err)
}
if resp.GetError() != "" {
t.Fatalf("ReceiveFile shard %d error response: %s", s.shardID, resp.GetError())
}
if resp.GetBytesWritten() != uint64(len(payload)) {
t.Fatalf("ReceiveFile shard %d bytes_written mismatch: got %d want %d",
s.shardID, resp.GetBytesWritten(), len(payload))
}
}
// Scan every data dir and record which shard files live where.
perDirShards := make(map[int][]uint32)
for dirIdx, dir := range dataDirs {
for _, s := range shards {
shardPath := filepath.Join(dir, fmt.Sprintf("%s_%d%s", collection, volumeID, shardExt(s.shardID)))
if _, err := os.Stat(shardPath); err == nil {
perDirShards[dirIdx] = append(perDirShards[dirIdx], s.shardID)
} else if !os.IsNotExist(err) {
t.Fatalf("unexpected stat error for %s: %v", shardPath, err)
}
}
}
for _, s := range shards {
found := false
for _, got := range perDirShards[s.expectedDirIdx] {
if got == s.shardID {
found = true
break
}
}
if !found {
t.Fatalf("shard %d (requestedDisk=%d) expected on dir[%d], full layout: %v",
s.shardID, s.requestedDisk, s.expectedDirIdx, perDirShards)
}
for dirIdx, ids := range perDirShards {
if dirIdx == s.expectedDirIdx {
continue
}
for _, id := range ids {
if id == s.shardID {
t.Fatalf("shard %d leaked onto dir[%d]: full layout: %v", s.shardID, dirIdx, perDirShards)
}
}
}
}
t.Logf("disk_id honored: shard placement = %v", perDirShards)
}
func TestReceiveFileEcShardRejectsInvalidDiskID(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const dataDirCount = 2
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
const volumeID = uint32(91840)
const collection = "ec-invalid-disk"
payload := []byte("invalid-disk-id-payload")
stream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create: %v", err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: ".ec00",
Collection: collection,
IsEcVolume: true,
ShardId: 0,
FileSize: uint64(len(payload)),
DiskId: 99,
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info: %v", err)
}
resp, err := stream.CloseAndRecv()
if err != nil {
t.Fatalf("ReceiveFile close: %v", err)
}
if resp.GetError() == "" {
t.Fatalf("expected invalid disk_id rejection, got success response: %+v", resp)
}
}
test/volume_server/grpc/ec_receive_truncates_mounted_test.go
+186
View File
@@ -0,0 +1,186 @@
package volume_server_grpc_test
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
func TestReceiveFileRejectsOverwriteOfMountedEcShard(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(91841)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 918401, 0x9184CAFE)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("ec-receive-overwrite-content-for-issue-9184-repro"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: "",
}); err != nil {
t.Fatalf("VolumeEcShardsGenerate: %v", err)
}
if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: "",
ShardIds: []uint32{0},
}); err != nil {
t.Fatalf("VolumeEcShardsMount: %v", err)
}
dataDir := filepath.Join(clusterHarness.BaseDir(), "volume")
shardPath := filepath.Join(dataDir, fmt.Sprintf("%d.ec00", volumeID))
origInfo, err := os.Stat(shardPath)
if err != nil {
t.Fatalf("stat mounted shard %s: %v", shardPath, err)
}
origSize := origInfo.Size()
if origSize == 0 {
t.Fatalf("mounted shard %s unexpectedly empty", shardPath)
}
readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
VolumeId: volumeID,
ShardId: 0,
Offset: 0,
Size: 1,
})
if err != nil {
t.Fatalf("VolumeEcShardRead (pre): %v", err)
}
if _, err := readStream.Recv(); err != nil {
t.Fatalf("VolumeEcShardRead Recv (pre): %v", err)
}
overwritePayload := []byte("bug-9184-overwrite")
receiveStream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create: %v", err)
}
if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: ".ec00",
Collection: "",
IsEcVolume: true,
ShardId: 0,
FileSize: uint64(len(overwritePayload)),
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info: %v", err)
}
resp, err := receiveStream.CloseAndRecv()
if err != nil {
t.Logf("ReceiveFile rejected at stream level: %v", err)
} else {
if resp.GetError() == "" {
t.Fatalf("expected ReceiveFile to reject overwrite of mounted shard, got success: %+v", resp)
}
if !strings.Contains(resp.GetError(), "mounted") {
t.Fatalf("expected error to mention mounted; got: %s", resp.GetError())
}
}
afterInfo, err := os.Stat(shardPath)
if err != nil {
t.Fatalf("stat shard after rejected overwrite: %v", err)
}
if afterInfo.Size() != origSize {
t.Fatalf("shard %s was modified despite rejection: size was %d, now %d",
shardPath, origSize, afterInfo.Size())
}
postStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
VolumeId: volumeID,
ShardId: 0,
Offset: 0,
Size: 1,
})
if err != nil {
t.Fatalf("VolumeEcShardRead (post): %v", err)
}
if _, err := postStream.Recv(); err != nil {
t.Fatalf("VolumeEcShardRead Recv (post): %v", err)
}
t.Logf("ReceiveFile correctly refused overwrite; mounted shard intact at %d bytes", afterInfo.Size())
}
func TestReceiveFileAllowsEcShardWhenNoMount(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
const volumeID = uint32(91843)
const collection = "ec-receive-no-mount"
payload := []byte("ok-to-receive-not-mounted")
stream, err := grpcClient.ReceiveFile(ctx)
if err != nil {
t.Fatalf("ReceiveFile stream create: %v", err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: ".ec00",
Collection: collection,
IsEcVolume: true,
ShardId: 0,
FileSize: uint64(len(payload)),
},
},
}); err != nil {
t.Fatalf("ReceiveFile send info: %v", err)
}
if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payload},
}); err != nil {
t.Fatalf("ReceiveFile send content: %v", err)
}
resp, err := stream.CloseAndRecv()
if err != nil {
t.Fatalf("ReceiveFile close: %v", err)
}
if resp.GetError() != "" {
t.Fatalf("expected success on unmounted volume, got error: %s", resp.GetError())
}
if resp.GetBytesWritten() != uint64(len(payload)) {
t.Fatalf("bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(payload))
}
}
test/volume_server/grpc/ec_verify_multi_disk_test.go
+182
View File
@@ -0,0 +1,182 @@
package volume_server_grpc_test
import (
"context"
"net/http"
"sort"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks drives the full path
// behind the ec.encode source-deletion gate. A multi-disk volume server
// ends up with EC shards split across disks (each registers its own
// EcVolume entry in DiskLocation.ecVolumes), and the volume server's
// VolumeEcShardsInfo RPC must walk every DiskLocation rather than
// reporting whichever disk Store.FindEcVolume picks first.
//
// Pre-fix, verifyEcShardsBeforeDelete refused to delete source volumes —
// the shard-bitmap union across destinations fell short of dataShards +
// parityShards because each destination only reported shards on one of
// its disks. With the handler fix, the same VerifyShardsAcrossServers
// call returns a complete bitmap and the gate opens.
func TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const (
dataDirCount = 2
volumeID = uint32(9558)
collection = "ec-multi-disk-verify"
)
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
dataDirs := clusterHarness.VolumeDataDirs()
if len(dataDirs) != dataDirCount {
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
}
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
needles := []struct {
fid string
payload []byte
}{
{framework.NewFileID(volumeID, 9559, 0xC0FFEE01), bytesOfLen(64, 0xB1)},
{framework.NewFileID(volumeID, 9560, 0xC0FFEE02), bytesOfLen(8192, 0xB2)},
{framework.NewFileID(volumeID, 9561, 0xC0FFEE03), bytesOfLen(131072, 0xB3)},
}
for _, n := range needles {
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
}
}
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: collection,
}); err != nil {
t.Fatalf("VolumeEcShardsGenerate: %v", err)
}
// Generate places every shard plus the .ecx/.ecj/.vif on the .dat's
// disk (disk 0). Mount all 14 there first so the next step's restart
// has a steady starting state.
allShards := make([]uint32, erasure_coding.TotalShardsCount)
for i := range allShards {
allShards[i] = uint32(i)
}
if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: collection,
ShardIds: allShards,
}); err != nil {
t.Fatalf("VolumeEcShardsMount all shards: %v", err)
}
// Drop the .dat so the EC shards are the only data path — mirrors the
// real ec.encode flow before verifyEcShardsBeforeDelete fires.
if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
VolumeId: volumeID,
}); err != nil {
t.Fatalf("VolumeDelete (drop .dat): %v", err)
}
// Move half the shards onto disk 1, leaving .ecx on disk 0. After
// restart, the cross-disk reconcile path attaches each disk's shards
// against its own EcVolume entry — the exact in-memory shape the bug
// reporter saw on a multi-disk destination.
clusterHarness.StopVolumeServer()
const splitAt = 7
for shard := 0; shard < splitAt; shard++ {
movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
}
if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
t.Fatalf("setup: .ecx must stay on disk 0 to exercise the multi-disk path")
}
clusterHarness.RestartVolumeServer()
conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn2.Close()
postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
}
if len(postReconcileLayout[0]) == 0 || len(postReconcileLayout[1]) == 0 {
t.Fatalf("post-reconcile: expected shards on BOTH disks, got per-disk layout %v", postReconcileLayout)
}
// Direct RPC assertion: VolumeEcShardsInfo must report every shard
// the server holds, not just the ones registered against the first
// matching DiskLocation.
infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("VolumeEcShardsInfo: %v", err)
}
gotShardIds := make([]int, 0, len(infoResp.GetEcShardInfos()))
for _, info := range infoResp.GetEcShardInfos() {
if info.GetVolumeId() != volumeID {
t.Errorf("EcShardInfo VolumeId=%d, want %d", info.GetVolumeId(), volumeID)
}
gotShardIds = append(gotShardIds, int(info.GetShardId()))
}
sort.Ints(gotShardIds)
wantShardIds := make([]int, erasure_coding.TotalShardsCount)
for i := range wantShardIds {
wantShardIds[i] = i
}
if len(gotShardIds) != len(wantShardIds) {
t.Fatalf("VolumeEcShardsInfo returned %d shards (ids=%v), want %d (ids=%v) — per-disk layout=%v",
len(gotShardIds), gotShardIds, len(wantShardIds), wantShardIds, postReconcileLayout)
}
for i, sid := range wantShardIds {
if gotShardIds[i] != sid {
t.Fatalf("VolumeEcShardsInfo shard ids=%v, want %v (per-disk layout=%v)",
gotShardIds, wantShardIds, postReconcileLayout)
}
}
// End-to-end assertion via the same helper the worker uses to gate
// source-volume deletion (weed/worker/tasks/erasure_coding/ec_task.go
// verifyEcShardsBeforeDelete). The union across destinations is what
// RequireRecoverableShardSet measures; with one destination that holds
// every shard, the union must cover dataShards + parityShards without
// being reported as degraded.
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
servers := []string{clusterHarness.VolumeServerAddress()}
union, perServer := erasure_coding.VerifyShardsAcrossServers(ctx, volumeID, servers, dialOption)
degraded, err := erasure_coding.RequireRecoverableShardSet(
volumeID, union, erasure_coding.DataShardsCount, erasure_coding.TotalShardsCount)
if err != nil {
t.Fatalf("verifyEcShardsBeforeDelete-equivalent gate failed: %v\nper-server inventory: %s\nper-disk layout: %v",
err, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
}
if degraded {
t.Fatalf("verifyEcShardsBeforeDelete-equivalent gate reported a degraded shard set\nper-server inventory: %s\nper-disk layout: %v",
erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
}
if got, want := union.Count(), erasure_coding.TotalShardsCount; got != want {
t.Fatalf("VerifyShardsAcrossServers union covered %d/%d shards (per-server=%s, layout=%v)",
got, want, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
}
}
test/volume_server/grpc/erasure_coding_test.go
+120 -26
View File
@@ -23,28 +23,31 @@ func TestEcMaintenanceModeRejections(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
// Encode a volume before entering maintenance mode so the shard delete
// below has real shards to remove.
const ecVolumeID = uint32(1)
framework.AllocateVolume(t, grpcClient, ecVolumeID, "")
httpClient := framework.NewHTTPClient()
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(ecVolumeID, 990001, 0x11223344), []byte("ec-maintenance-content"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{
Maintenance: true,
Version: stateResp.GetState().GetVersion(),
},
})
if err != nil {
t.Fatalf("SetState maintenance=true failed: %v", err)
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: ecVolumeID, Collection: ""}); err != nil {
t.Fatalf("VolumeEcShardsGenerate before maintenance failed: %v", err)
}
_, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
framework.EnableMaintenanceMode(t, ctx, grpcClient)
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err)
}
@@ -59,13 +62,23 @@ func TestEcMaintenanceModeRejections(t *testing.T) {
t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err)
}
// Deleting shards removes data rather than adding it, and is how evacuating
// EC shards off a server in maintenance mode finishes (issue #11066).
_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: 1,
VolumeId: ecVolumeID,
Collection: "",
ShardIds: []uint32{0},
})
if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err)
if err != nil {
t.Fatalf("VolumeEcShardsDelete should succeed in maintenance mode, got: %v", err)
}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: ecVolumeID,
Collection: "",
ShardIds: []uint32{0},
})
if err == nil {
t.Fatalf("VolumeEcShardsMount should fail once shard 0 has been deleted")
}
_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
@@ -92,7 +105,7 @@ func TestEcMissingInvalidAndNoopPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -199,7 +212,7 @@ func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -268,7 +281,7 @@ func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -369,7 +382,7 @@ func TestEcRebuildMissingShardLifecycle(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -449,7 +462,7 @@ func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -560,7 +573,7 @@ func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -621,7 +634,7 @@ func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -752,12 +765,93 @@ func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
}
}
// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
// can be read back correctly from EC shards after VolumeEcShardsGenerate.
// This catches the race condition fixed in this PR where .ecx could reference
// data not present in EC shards.
func TestEcIndexConsistencyAfterEncode(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(130)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
// Upload multiple needles of varying sizes
type testNeedle struct {
fid string
payload []byte
}
needles := []testNeedle{
{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)}, // 1KB
{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)}, // 64KB
{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)}, // 256KB
{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
}
// Fill larger payloads with recognizable data
for i := range needles {
for j := range needles[i].payload {
needles[i].payload[j] = byte(i*37 + j%251)
}
}
for _, n := range needles {
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// EC encode
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
}
// Mount all data shards so reads go through the EC path
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: "",
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount failed: %v", err)
}
// Read every needle back from EC shards and verify payload
for _, n := range needles {
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
readBody := framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
}
if string(readBody) != string(n.payload) {
t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
}
}
}
func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/fetch_remote_s3_test.go
+271
View File
@@ -0,0 +1,271 @@
package volume_server_grpc_test
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// findAvailablePort finds a free TCP port on localhost.
// startWeedMini starts a weed mini subprocess and returns the S3 endpoint and cleanup func.
func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
t.Helper()
weedBin, err := exec.LookPath("weed")
if err != nil {
weedBin = filepath.Join("..", "..", "..", "weed", "weed_binary")
if _, err := os.Stat(weedBin); os.IsNotExist(err) {
t.Skip("weed binary not found, skipping S3 remote storage test")
}
}
miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3", "Admin"})
miniMasterPort := miniPorts[0]
miniVolumePort := miniPorts[1]
miniFilerPort := miniPorts[2]
miniS3Port := miniPorts[3]
miniAdminPort := miniPorts[4]
miniDir := t.TempDir()
os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644)
ctx, cancel := context.WithCancel(context.Background())
miniArgs := append([]string{"mini",
fmt.Sprintf("-dir=%s", miniDir),
fmt.Sprintf("-master.port=%d", miniMasterPort),
fmt.Sprintf("-volume.port=%d", miniVolumePort),
fmt.Sprintf("-filer.port=%d", miniFilerPort),
fmt.Sprintf("-s3.port=%d", miniS3Port),
"-webdav=false",
"-s3.port.lance=0",
"-s3.port.iceberg=0",
}, testutil.MiniAdminArgs(miniAdminPort)...)
miniCmd := exec.CommandContext(ctx, weedBin, miniArgs...)
miniCmd.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID=admin", "AWS_SECRET_ACCESS_KEY=admin")
miniCmd.Dir = miniDir
logFile, _ := os.CreateTemp("", "weed-mini-*.log")
miniCmd.Stdout = logFile
miniCmd.Stderr = logFile
t.Logf("weed mini logs at %s", logFile.Name())
if err := miniCmd.Start(); err != nil {
cancel()
logFile.Close()
t.Fatalf("start weed mini: %v", err)
}
if !testutil.WaitForPort(miniS3Port, 30*time.Second) {
cancel()
miniCmd.Wait()
logFile.Close()
t.Fatalf("weed mini S3 not ready on port %d", miniS3Port)
}
t.Logf("weed mini S3 ready on port %d", miniS3Port)
return fmt.Sprintf("http://127.0.0.1:%d", miniS3Port), func() {
cancel()
miniCmd.Wait()
logFile.Close()
}
}
func newS3Client(endpoint string) *s3.S3 {
sess, _ := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(endpoint),
Credentials: credentials.NewStaticCredentials("admin", "admin", ""),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
})
return s3.New(sess)
}
// TestFetchAndWriteNeedleFromS3 tests the full FetchAndWriteNeedle flow:
// 1. Start a weed mini instance as S3 backend
// 2. Upload a test object to it via S3 API
// 3. Call FetchAndWriteNeedle on the volume server to fetch from S3
// 4. Verify the response contains a valid e_tag
func TestFetchAndWriteNeedleFromS3(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
s3Endpoint, cleanupMini := startWeedMini(t)
defer cleanupMini()
s3Client := newS3Client(s3Endpoint)
// Create bucket and upload test data
bucket := "test-remote-fetch"
s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
testData := []byte("Hello from S3 remote storage! This is test data for FetchAndWriteNeedle.")
testKey := "test-object.dat"
_, err := s3Client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(testKey),
Body: bytes.NewReader(testData),
})
if err != nil {
t.Fatalf("put object: %v", err)
}
t.Logf("uploaded %d bytes to s3://%s/%s", len(testData), bucket, testKey)
// Start volume server
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(99)
framework.AllocateVolume(t, grpcClient, volumeID, "")
grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer grpcCancel()
// FetchAndWriteNeedle from S3
resp, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
VolumeId: volumeID,
NeedleId: 42,
Cookie: 12345,
Offset: 0,
Size: int64(len(testData)),
RemoteConf: &remote_pb.RemoteConf{
Name: "test-s3",
Type: "s3",
S3AccessKey: "admin",
S3SecretKey: "admin",
S3Region: "us-east-1",
S3Endpoint: s3Endpoint,
S3ForcePathStyle: true,
},
RemoteLocation: &remote_pb.RemoteStorageLocation{
Name: "test-s3",
Bucket: bucket,
Path: "/" + testKey,
},
})
if err != nil {
t.Fatalf("FetchAndWriteNeedle failed: %v", err)
}
if resp.GetETag() == "" {
t.Fatal("FetchAndWriteNeedle returned empty e_tag")
}
t.Logf("FetchAndWriteNeedle success: e_tag=%s", resp.GetETag())
}
// TestFetchAndWriteNeedleFromS3WithPartialRead tests reading a byte range from S3.
func TestFetchAndWriteNeedleFromS3WithPartialRead(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
s3Endpoint, cleanupMini := startWeedMini(t)
defer cleanupMini()
s3Client := newS3Client(s3Endpoint)
bucket := "partial-read-test"
s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
// Upload 1KB of data
fullData := make([]byte, 1024)
for i := range fullData {
fullData[i] = byte(i % 256)
}
s3Client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket), Key: aws.String("big.dat"),
Body: bytes.NewReader(fullData),
})
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
framework.AllocateVolume(t, grpcClient, 98, "")
grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer grpcCancel()
// Fetch only bytes 100-199 (100 bytes) from the 1KB object
resp, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
VolumeId: 98, NeedleId: 7, Cookie: 999,
Offset: 100, Size: 100,
RemoteConf: &remote_pb.RemoteConf{
Name: "test-s3-partial", Type: "s3",
S3AccessKey: "admin", S3SecretKey: "admin",
S3Region: "us-east-1", S3Endpoint: s3Endpoint, S3ForcePathStyle: true,
},
RemoteLocation: &remote_pb.RemoteStorageLocation{
Name: "test-s3-partial", Bucket: bucket, Path: "/big.dat",
},
})
if err != nil {
t.Fatalf("FetchAndWriteNeedle partial read failed: %v", err)
}
if resp.GetETag() == "" {
t.Fatal("empty e_tag for partial read")
}
t.Logf("FetchAndWriteNeedle partial read success: e_tag=%s", resp.GetETag())
}
// TestFetchAndWriteNeedleS3NotFound tests that fetching a non-existent S3 object returns an error.
func TestFetchAndWriteNeedleS3NotFound(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
s3Endpoint, cleanupMini := startWeedMini(t)
defer cleanupMini()
s3Client := newS3Client(s3Endpoint)
bucket := "notfound-test"
s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
framework.AllocateVolume(t, grpcClient, 97, "")
grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer grpcCancel()
_, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
VolumeId: 97, NeedleId: 1, Cookie: 1,
Offset: 0, Size: 100,
RemoteConf: &remote_pb.RemoteConf{
Name: "test-s3-nf", Type: "s3",
S3AccessKey: "admin", S3SecretKey: "admin",
S3Region: "us-east-1", S3Endpoint: s3Endpoint, S3ForcePathStyle: true,
},
RemoteLocation: &remote_pb.RemoteStorageLocation{
Name: "test-s3-nf", Bucket: bucket, Path: "/does-not-exist.dat",
},
})
if err == nil {
t.Fatal("FetchAndWriteNeedle should fail for non-existent object")
}
if !strings.Contains(err.Error(), "read from remote") {
t.Fatalf("expected 'read from remote' error, got: %v", err)
}
t.Logf("correctly got error for non-existent object: %v", err)
}
test/volume_server/grpc/health_state_test.go
+2 -2
View File
@@ -16,7 +16,7 @@ func TestStateAndStatusRPCs(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -94,7 +94,7 @@ func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/mixed_balance_test.go
+567
View File
@@ -0,0 +1,567 @@
package volume_server_grpc_test
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
// volumeURL, asserts a successful response, then polls with GET until
// the file returns 404 (tombstone visible) or the timeout elapses.
func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
t.Helper()
req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
if err != nil {
t.Fatalf("build delete request for %s: %v", fid, err)
}
resp := framework.DoRequest(t, httpClient, req)
framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
}
// Poll until GET returns 404 (tombstone flushed to disk).
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
status := getResp.StatusCode
framework.ReadAllAndClose(t, getResp)
if status == http.StatusNotFound {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("delete %s: tombstone not visible after 5s", fid)
}
// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
// server to a Rust volume server. This is the core operation behind volume
// balancing in a mixed Go+Rust cluster.
func TestMixedBalanceCopyGoToRust(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
// server 0 = Go, server 1 = Rust
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
httpClient := framework.NewHTTPClient()
const volumeID = uint32(50)
// Allocate volume on Go server and upload test data
framework.AllocateVolume(t, goClient, volumeID, "")
testFiles := []struct {
key uint64
cookie uint32
data []byte
}{
{1, 0xAABBCCDD, []byte("hello from Go server")},
{2, 0x11223344, []byte("second file for balance test")},
{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
}
for _, f := range testFiles {
fid := framework.NewFileID(volumeID, f.key, f.cookie)
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Read source volume status before copy
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
}
t.Logf("Source: dat=%d idx=%d files=%d version=%d",
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
sourceStatus.GetFileCount(), sourceStatus.GetVersion())
// Copy volume from Go (server 0) to Rust (server 1)
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: cluster.VolumeServerAddress(0),
})
if err != nil {
t.Fatalf("VolumeCopy call failed: %v", err)
}
var lastAppendAtNs uint64
for {
resp, recvErr := copyStream.Recv()
if recvErr != nil {
if recvErr != io.EOF {
t.Fatalf("VolumeCopy recv error: %v", recvErr)
}
break
}
if resp.GetLastAppendAtNs() != 0 {
lastAppendAtNs = resp.GetLastAppendAtNs()
}
}
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
// Verify: read volume status from Rust server
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
}
t.Logf("Target: dat=%d idx=%d files=%d version=%d",
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
targetStatus.GetFileCount(), targetStatus.GetVersion())
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
t.Fatalf("dat file size mismatch: source=%d target=%d",
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
}
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
t.Fatalf("idx file size mismatch: source=%d target=%d",
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
}
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
t.Fatalf("file count mismatch: source=%d target=%d",
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
}
// Verify data can be read from Rust server
for _, f := range testFiles {
fid := framework.NewFileID(volumeID, f.key, f.cookie)
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
}
if !bytes.Equal(body, f.data) {
t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
}
}
}
// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
// volume server to a Go volume server.
func TestMixedBalanceCopyRustToGo(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
httpClient := framework.NewHTTPClient()
const volumeID = uint32(51)
// Allocate volume on Rust server and upload test data
framework.AllocateVolume(t, rustClient, volumeID, "")
testFiles := []struct {
key uint64
cookie uint32
data []byte
}{
{1, 0xAABBCCDD, []byte("hello from Rust server")},
{2, 0x11223344, []byte("second file for reverse balance")},
}
for _, f := range testFiles {
fid := framework.NewFileID(volumeID, f.key, f.cookie)
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
}
// Copy volume from Rust (server 1) to Go (server 0)
copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: cluster.VolumeServerAddress(1),
})
if err != nil {
t.Fatalf("VolumeCopy call failed: %v", err)
}
for {
_, recvErr := copyStream.Recv()
if recvErr != nil {
if recvErr != io.EOF {
t.Fatalf("VolumeCopy recv error: %v", recvErr)
}
break
}
}
targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
}
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
t.Fatalf("dat file size mismatch: source=%d target=%d",
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
}
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
t.Fatalf("idx file size mismatch: source=%d target=%d",
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
}
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
t.Fatalf("file count mismatch: source=%d target=%d",
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
}
// Verify data can be read from Go server
for _, f := range testFiles {
fid := framework.NewFileID(volumeID, f.key, f.cookie)
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
}
if !bytes.Equal(body, f.data) {
t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
}
}
}
// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
// volumes that have both active and deleted needles.
func TestMixedBalanceCopyWithDeletes(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
httpClient := framework.NewHTTPClient()
const volumeID = uint32(52)
framework.AllocateVolume(t, goClient, volumeID, "")
// Upload files
for i := uint64(1); i <= 5; i++ {
fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
}
}
// Delete some files and wait for tombstones to be visible
for _, key := range []uint64{2, 4} {
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus: %v", err)
}
t.Logf("Source after deletes: dat=%d idx=%d files=%d",
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
sourceStatus.GetFileCount())
// Copy volume from Go to Rust
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: cluster.VolumeServerAddress(0),
})
if err != nil {
t.Fatalf("VolumeCopy: %v", err)
}
var lastAppendAtNs uint64
for {
resp, recvErr := copyStream.Recv()
if recvErr != nil {
if recvErr != io.EOF {
t.Fatalf("VolumeCopy recv error: %v", recvErr)
}
break
}
if resp.GetLastAppendAtNs() != 0 {
lastAppendAtNs = resp.GetLastAppendAtNs()
}
}
if lastAppendAtNs == 0 {
t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
}
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
}
t.Logf("Target after copy: dat=%d idx=%d files=%d",
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
targetStatusAfterCopy.GetFileCount())
if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
t.Fatalf("dat file size mismatch: source=%d target=%d",
sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
}
if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
t.Fatalf("idx file size mismatch: source=%d target=%d",
sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
}
// Tail from the copy checkpoint — source is unchanged (deletes happened
// before copy), so tailing should not append any data.
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
VolumeId: volumeID,
SinceNs: lastAppendAtNs,
IdleTimeoutSeconds: 3,
SourceVolumeServer: cluster.VolumeServerAddress(0),
})
if err != nil {
t.Fatalf("VolumeTailReceiver: %v", err)
}
targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
}
if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
t.Fatalf("dat grew after tail: before=%d after=%d",
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
}
if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
t.Fatalf("idx grew after tail: before=%d after=%d",
targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
}
// Verify surviving files are readable from Rust
for _, key := range []uint64{1, 3, 5} {
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
}
expected := fmt.Sprintf("file-%d", key)
if string(body) != expected {
t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
}
}
// Verify deleted files return 404
for _, key := range []uint64{2, 4} {
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
}
}
}
// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
// flow: mark readonly → copy → tail → verify sizes → delete source.
// This mirrors the steps in balance_task.go Execute().
func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
httpClient := framework.NewHTTPClient()
const volumeID = uint32(53)
framework.AllocateVolume(t, goClient, volumeID, "")
// Upload test data
for i := uint64(1); i <= 5; i++ {
fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Step 1: Copy volume to Rust server (source stays writable so we can
// delete after copy to exercise the tail tombstone path)
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: cluster.VolumeServerAddress(0),
})
if err != nil {
t.Fatalf("VolumeCopy: %v", err)
}
var lastAppendAtNs uint64
for {
resp, recvErr := copyStream.Recv()
if recvErr != nil {
if recvErr != io.EOF {
t.Fatalf("VolumeCopy recv: %v", recvErr)
}
break
}
if resp.GetLastAppendAtNs() != 0 {
lastAppendAtNs = resp.GetLastAppendAtNs()
}
}
t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
// Step 2: Delete file 3 on the source AFTER copy. This creates a
// tombstone needle that the tail step must propagate to the Rust server.
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
framework.NewFileID(volumeID, 3, 0xABCD0003))
// Step 3: Mark source readonly so no further writes arrive during tail
_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("mark readonly: %v", err)
}
// Read source status (the reference for post-tail verification)
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("read source status: %v", err)
}
t.Logf("Source: dat=%d idx=%d files=%d",
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
sourceStatus.GetFileCount())
// Step 4: Tail for updates — this must pick up the delete tombstone
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
VolumeId: volumeID,
SinceNs: lastAppendAtNs,
IdleTimeoutSeconds: 5,
SourceVolumeServer: cluster.VolumeServerAddress(0),
})
if err != nil {
t.Fatalf("VolumeTailReceiver: %v", err)
}
// Step 5: Verify dat/idx sizes match after tail.
// We compare file sizes (byte-level correctness) rather than file_count
// because the tail writes the tombstone via the write path (which
// increments file_count) while the source used the delete path (which
// only increments deletion_count). This is consistent with Go behavior.
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("read target status: %v", err)
}
t.Logf("Target: dat=%d idx=%d files=%d",
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
targetStatus.GetFileCount())
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
t.Fatalf("dat size mismatch after tail: source=%d target=%d",
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
}
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
t.Fatalf("idx size mismatch after tail: source=%d target=%d",
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
}
// Step 6: Delete volume from source
_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("delete source volume: %v", err)
}
// Verify source volume is gone
_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: volumeID,
})
if err == nil {
t.Fatalf("expected error reading deleted source volume, got nil")
}
t.Logf("Source volume deleted successfully (error as expected: %v)", err)
// Verify all surviving data is readable from Rust (the new home)
for _, key := range []uint64{1, 2, 4, 5} {
fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
}
expected := fmt.Sprintf("balance-move-file-%d", key)
if string(body) != expected {
t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
}
}
// Verify deleted file is 404 on Rust (tombstone propagated via tail)
{
fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
}
}
t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
}
test/volume_server/grpc/move_tail_timestamp_test.go
+285
View File
@@ -0,0 +1,285 @@
package volume_server_grpc_test
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestVolumeCopyReturnsPreciseLastAppendTimestamp(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
sourceConn, sourceClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer sourceConn.Close()
destConn, destClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer destConn.Close()
const volumeID = uint32(999)
framework.AllocateVolume(t, sourceClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 1, 0x42)
payload := []byte("move-tail-timestamp-payload")
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("source upload failed: status %d", uploadResp.StatusCode)
}
sourceDir := filepath.Join(cluster.BaseDir(), "volume0")
datPath := storage.VolumeFileName(sourceDir, "", int(volumeID)) + ".dat"
futureTime := time.Now().Add(2 * time.Hour)
if err := os.Chtimes(datPath, futureTime, futureTime); err != nil {
t.Fatalf("set future dat timestamp: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
sourceDataNode := cluster.VolumeAdminAddress(0) + "." + strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: sourceDataNode,
})
if err != nil {
t.Fatalf("VolumeCopy start failed: %v", err)
}
var lastAppendAtNs uint64
for {
resp, recvErr := copyStream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
t.Fatalf("VolumeCopy recv failed: %v", recvErr)
}
if ts := resp.GetLastAppendAtNs(); ts > 0 {
lastAppendAtNs = ts
}
}
if lastAppendAtNs == 0 {
t.Fatalf("volume copy did not return a last append timestamp")
}
destDir := filepath.Join(cluster.BaseDir(), "volume1")
actualLastAppend := readLastAppendAtNs(t, destDir, volumeID)
if actualLastAppend == 0 {
t.Fatalf("failed to compute last append timestamp from destination files")
}
if lastAppendAtNs != actualLastAppend {
t.Fatalf("last append timestamp mismatch: got %d, actual %d", lastAppendAtNs, actualLastAppend)
}
}
func readLastAppendAtNs(t testing.TB, volumeDir string, volumeID uint32) uint64 {
t.Helper()
baseName := storage.VolumeFileName(volumeDir, "", int(volumeID))
idxPath := baseName + ".idx"
idxFile, err := os.Open(idxPath)
if err != nil {
t.Fatalf("open idx file %s: %v", idxPath, err)
}
defer idxFile.Close()
stat, err := idxFile.Stat()
if err != nil {
t.Fatalf("stat idx file %s: %v", idxPath, err)
}
if stat.Size() == 0 {
return 0
}
if stat.Size()%int64(types.NeedleMapEntrySize) != 0 {
t.Fatalf("unexpected idx file size %d", stat.Size())
}
buf := make([]byte, types.NeedleMapEntrySize)
if _, err := idxFile.ReadAt(buf, stat.Size()-int64(types.NeedleMapEntrySize)); err != nil {
t.Fatalf("read idx entry: %v", err)
}
_, offset, _ := idx.IdxFileEntry(buf)
if offset.IsZero() {
return 0
}
datPath := baseName + ".dat"
datFile, err := os.Open(datPath)
if err != nil {
t.Fatalf("open dat file %s: %v", datPath, err)
}
defer datFile.Close()
datBackend := backend.NewDiskFile(datFile)
n, _, _, err := needle.ReadNeedleHeader(datBackend, needle.GetCurrentVersion(), offset.ToActualOffset())
if err != nil {
t.Fatalf("read needle header: %v", err)
}
tailOffset := offset.ToActualOffset() + int64(types.NeedleHeaderSize) + int64(n.Size)
tail := make([]byte, needle.NeedleChecksumSize+types.TimestampSize)
readCount, readErr := datBackend.ReadAt(tail, tailOffset)
if readErr == io.EOF && readCount == len(tail) {
readErr = nil
}
if readErr != nil {
t.Fatalf("read needle tail: %v", readErr)
}
return util.BytesToUint64(tail[needle.NeedleChecksumSize : needle.NeedleChecksumSize+types.TimestampSize])
}
func TestVolumeMoveHandlesInFlightWrites(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
sourceConn, sourceClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer sourceConn.Close()
destConn, destClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer destConn.Close()
const volumeID = uint32(988)
framework.AllocateVolume(t, sourceClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 7, 0xABCDEF01)
payload := []byte("volume-move-live-payload")
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("initial upload failed: %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
sourceDataNode := cluster.VolumeAdminAddress(0) + "." + strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
VolumeId: volumeID,
SourceDataNode: sourceDataNode,
})
if err != nil {
t.Fatalf("VolumeCopy start failed: %v", err)
}
var lastAppendAtNs uint64
for {
resp, recvErr := copyStream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
t.Fatalf("VolumeCopy recv failed: %v", recvErr)
}
if ts := resp.GetLastAppendAtNs(); ts > 0 {
lastAppendAtNs = ts
}
}
if lastAppendAtNs == 0 {
t.Fatalf("volume copy did not return a last append timestamp")
}
type written struct {
fid string
data []byte
}
var writesMu sync.Mutex
var writes []written
var uploadErr error
writeCtx, writeCancel := context.WithCancel(context.Background())
var writerWG sync.WaitGroup
writerWG.Add(1)
go func() {
defer writerWG.Done()
client := framework.NewHTTPClient()
for i := 0; i < 12; i++ {
select {
case <-writeCtx.Done():
return
default:
}
livePayload := []byte("live-data-" + fmt.Sprintf("%02d", i))
liveFid := framework.NewFileID(volumeID, uint64(2000+i), 0xAAAA1111+uint32(i))
resp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(0), liveFid, livePayload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
uploadErr = fmt.Errorf("live upload failed: %d", resp.StatusCode)
return
}
writesMu.Lock()
writes = append(writes, written{fid: liveFid, data: livePayload})
writesMu.Unlock()
time.Sleep(250 * time.Millisecond)
}
}()
tailCtx, tailCancel := context.WithTimeout(context.Background(), 60*time.Second)
defer tailCancel()
_, err = destClient.VolumeTailReceiver(tailCtx, &volume_server_pb.VolumeTailReceiverRequest{
VolumeId: volumeID,
SourceVolumeServer: sourceDataNode,
SinceNs: lastAppendAtNs,
IdleTimeoutSeconds: 3,
})
if err != nil {
writeCancel()
writerWG.Wait()
t.Fatalf("VolumeTailReceiver failed: %v", err)
}
writeCancel()
writerWG.Wait()
if uploadErr != nil {
t.Fatalf("upload goroutine error: %v", uploadErr)
}
writesMu.Lock()
sampleCount := len(writes)
if sampleCount == 0 {
writesMu.Unlock()
t.Fatal("no live writes captured")
}
sample := writes
if sampleCount > 3 {
sample = writes[sampleCount-3:]
}
writesMu.Unlock()
httpCheckClient := framework.NewHTTPClient()
for _, w := range sample {
resp := framework.ReadBytes(t, httpCheckClient, cluster.VolumeAdminURL(1), w.fid)
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("dest read %s status %d", w.fid, resp.StatusCode)
}
if !bytes.Equal(body, w.data) {
t.Fatalf("dest read body mismatch for %s: %q vs %q", w.fid, string(body), string(w.data))
}
}
}
test/volume_server/grpc/production_features_test.go
+338
View File
@@ -0,0 +1,338 @@
package volume_server_grpc_test
import (
"context"
"io"
"net/http"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func TestScrubVolumeDetectsHealthyData(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(101)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
needles := []struct {
needleID uint64
cookie uint32
body string
}{
{needleID: 1010001, cookie: 0xAA000001, body: "scrub-healthy-needle-one"},
{needleID: 1010002, cookie: 0xAA000002, body: "scrub-healthy-needle-two"},
{needleID: 1010003, cookie: 0xAA000003, body: "scrub-healthy-needle-three"},
}
for _, n := range needles {
fid := framework.NewFileID(volumeID, n.needleID, n.cookie)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte(n.body))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload needle %d expected 201, got %d", n.needleID, uploadResp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
scrubResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_FULL,
})
if err != nil {
t.Fatalf("ScrubVolume FULL mode failed: %v", err)
}
if scrubResp.GetTotalVolumes() != 1 {
t.Fatalf("ScrubVolume expected total_volumes=1, got %d", scrubResp.GetTotalVolumes())
}
if scrubResp.GetTotalFiles() < 3 {
t.Fatalf("ScrubVolume expected total_files >= 3, got %d", scrubResp.GetTotalFiles())
}
if len(scrubResp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("ScrubVolume expected no broken volumes for healthy data, got %v: %v", scrubResp.GetBrokenVolumeIds(), scrubResp.GetDetails())
}
}
func TestScrubVolumeLocalModeWithMultipleVolumes(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeIDA = uint32(102)
const volumeIDB = uint32(103)
framework.AllocateVolume(t, grpcClient, volumeIDA, "")
framework.AllocateVolume(t, grpcClient, volumeIDB, "")
httpClient := framework.NewHTTPClient()
fidA := framework.NewFileID(volumeIDA, 1020001, 0xBB000001)
uploadA := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fidA, []byte("scrub-local-vol-a"))
_ = framework.ReadAllAndClose(t, uploadA)
if uploadA.StatusCode != http.StatusCreated {
t.Fatalf("upload to volume A expected 201, got %d", uploadA.StatusCode)
}
fidB := framework.NewFileID(volumeIDB, 1030001, 0xBB000002)
uploadB := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fidB, []byte("scrub-local-vol-b"))
_ = framework.ReadAllAndClose(t, uploadB)
if uploadB.StatusCode != http.StatusCreated {
t.Fatalf("upload to volume B expected 201, got %d", uploadB.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
scrubResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
Mode: volume_server_pb.VolumeScrubMode_LOCAL,
})
if err != nil {
t.Fatalf("ScrubVolume LOCAL auto-select failed: %v", err)
}
if scrubResp.GetTotalVolumes() < 2 {
t.Fatalf("ScrubVolume LOCAL expected total_volumes >= 2, got %d", scrubResp.GetTotalVolumes())
}
if len(scrubResp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("ScrubVolume LOCAL expected no broken volumes, got %v: %v", scrubResp.GetBrokenVolumeIds(), scrubResp.GetDetails())
}
}
func TestVolumeServerStatusReturnsRealDiskStats(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
statusResp, err := grpcClient.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
if err != nil {
t.Fatalf("VolumeServerStatus failed: %v", err)
}
diskStatuses := statusResp.GetDiskStatuses()
if len(diskStatuses) == 0 {
t.Fatalf("VolumeServerStatus expected non-empty disk_statuses")
}
foundValid := false
for _, ds := range diskStatuses {
if ds.GetDir() != "" && ds.GetAll() > 0 && ds.GetFree() > 0 {
foundValid = true
break
}
}
if !foundValid {
t.Fatalf("VolumeServerStatus expected at least one disk status with Dir, All > 0, Free > 0; got %v", diskStatuses)
}
}
func TestReadNeedleBlobAndMetaVerifiesCookie(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(104)
const needleID = uint64(1040001)
const cookie = uint32(0xCC000001)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, needleID, cookie)
payload := []byte("read-needle-blob-meta-verify")
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("ReadVolumeFileStatus failed: %v", err)
}
if fileStatus.GetIdxFileSize() == 0 {
t.Fatalf("expected non-zero idx file size after upload")
}
idxBytes := prodCopyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".idx",
CompactionRevision: fileStatus.GetCompactionRevision(),
StopOffset: fileStatus.GetIdxFileSize(),
})
offset, size := prodFindNeedleOffsetAndSize(t, idxBytes, needleID)
blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
VolumeId: volumeID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleBlob failed: %v", err)
}
if len(blobResp.GetNeedleBlob()) == 0 {
t.Fatalf("ReadNeedleBlob returned empty blob")
}
metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
VolumeId: volumeID,
NeedleId: needleID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleMeta failed: %v", err)
}
if metaResp.GetCookie() != cookie {
t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), cookie)
}
if metaResp.GetCrc() == 0 {
t.Fatalf("ReadNeedleMeta expected non-zero CRC")
}
}
func TestBatchDeleteMultipleNeedles(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(105)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
type needle struct {
needleID uint64
cookie uint32
body string
fid string
}
needles := []needle{
{needleID: 1050001, cookie: 0xDD000001, body: "batch-del-needle-one"},
{needleID: 1050002, cookie: 0xDD000002, body: "batch-del-needle-two"},
{needleID: 1050003, cookie: 0xDD000003, body: "batch-del-needle-three"},
}
fids := make([]string, len(needles))
for i := range needles {
needles[i].fid = framework.NewFileID(volumeID, needles[i].needleID, needles[i].cookie)
fids[i] = needles[i].fid
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), needles[i].fid, []byte(needles[i].body))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload needle %d expected 201, got %d", needles[i].needleID, uploadResp.StatusCode)
}
}
// Verify all needles are readable before delete
for _, n := range needles {
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
_ = framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("pre-delete read of %s expected 200, got %d", n.fid, readResp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
deleteResp, err := grpcClient.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
FileIds: fids,
})
if err != nil {
t.Fatalf("BatchDelete failed: %v", err)
}
if len(deleteResp.GetResults()) != 3 {
t.Fatalf("BatchDelete expected 3 results, got %d", len(deleteResp.GetResults()))
}
for i, result := range deleteResp.GetResults() {
if result.GetStatus() != http.StatusAccepted {
t.Fatalf("BatchDelete result[%d] expected status 202, got %d (error: %s)", i, result.GetStatus(), result.GetError())
}
if result.GetSize() <= 0 {
t.Fatalf("BatchDelete result[%d] expected size > 0, got %d", i, result.GetSize())
}
}
// Verify all needles return 404 after delete
for _, n := range needles {
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
_ = framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusNotFound {
t.Fatalf("post-delete read of %s expected 404, got %d", n.fid, readResp.StatusCode)
}
}
}
// prodCopyFileBytes streams a CopyFile response into a byte slice.
func prodCopyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.CopyFile(ctx, req)
if err != nil {
t.Fatalf("CopyFile start failed: %v", err)
}
var out []byte
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
return out
}
if recvErr != nil {
t.Fatalf("CopyFile recv failed: %v", recvErr)
}
out = append(out, msg.GetFileContent()...)
}
}
// prodFindNeedleOffsetAndSize scans idx bytes for a needle's offset and size.
func prodFindNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
if entryOffset.IsZero() || entrySize <= 0 {
continue
}
return entryOffset.ToActualOffset(), int32(entrySize)
}
t.Fatalf("needle id %d not found in idx entries", needleID)
return 0, 0
}
test/volume_server/grpc/scrub_integration_test.go
+501
View File
@@ -0,0 +1,501 @@
package volume_server_grpc_test
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// --- Normal volume scrub tests ---
func TestScrubVolumeFullHealthy(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(200)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_FULL,
})
if err != nil {
t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
}
if resp.GetTotalVolumes() != 1 {
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
}
if resp.GetTotalFiles() != 3 {
t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
}
}
func TestScrubVolumeFullCorruptData(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(201)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_FULL,
})
if err != nil {
t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after data corruption, got none")
}
if len(resp.GetDetails()) == 0 {
t.Fatalf("expected error details for corrupt volume")
}
}
func TestScrubVolumeMixedHealthy(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const healthyVol = uint32(202)
const corruptVol = uint32(203)
framework.AllocateVolume(t, grpcClient, healthyVol, "")
framework.AllocateVolume(t, grpcClient, corruptVol, "")
httpClient := framework.NewHTTPClient()
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{healthyVol, corruptVol},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
})
if err != nil {
t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
}
if resp.GetTotalVolumes() != 2 {
t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
}
if len(resp.GetBrokenVolumeIds()) != 1 {
t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
}
if resp.GetBrokenVolumeIds()[0] != corruptVol {
t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
}
}
func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{99999},
Mode: volume_server_pb.VolumeScrubMode_FULL,
})
if err == nil {
t.Fatalf("ScrubVolume should fail for missing volume")
}
}
// --- EC volume scrub tests ---
// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
t.Helper()
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
}
allShards := make([]uint32, erasure_coding.TotalShardsCount)
for i := range allShards {
allShards[i] = uint32(i)
}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: "",
ShardIds: allShards,
})
if err != nil {
t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
}
}
func TestScrubEcVolumeIndexHealthy(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(210)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
})
if err != nil {
t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
}
if resp.GetTotalVolumes() != 1 {
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
}
}
func TestScrubEcVolumeLocalHealthy(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(211)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_LOCAL,
})
if err != nil {
t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
}
if resp.GetTotalVolumes() != 1 {
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
}
if resp.GetTotalFiles() != 1 {
t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
}
if len(resp.GetBrokenShardInfos()) != 0 {
t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
}
}
func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(212)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
// Corrupt shard 0 by truncating it.
framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_LOCAL,
})
if err != nil {
t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after shard corruption")
}
if len(resp.GetBrokenShardInfos()) == 0 {
t.Fatalf("expected broken shard info after shard corruption")
}
// Verify all reported broken shards belong to the corrupted volume.
for _, si := range resp.GetBrokenShardInfos() {
if si.GetVolumeId() != volumeID {
t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
}
}
}
func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeIDA = uint32(213)
const volumeIDB = uint32(214)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Auto-select (empty VolumeIds) should find both EC volumes.
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
Mode: volume_server_pb.VolumeScrubMode_INDEX,
})
if err != nil {
t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
}
if resp.GetTotalVolumes() < 2 {
t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
}
}
func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(215)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode(99),
})
if err == nil {
t.Fatalf("ScrubEcVolume should fail for unsupported mode")
}
}
func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(216)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Generate EC shards (this creates the .ecx file on disk).
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID,
Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
}
// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
// Now mount shards - the ecx file size will reflect the corruption.
allShards := make([]uint32, erasure_coding.TotalShardsCount)
for i := range allShards {
allShards[i] = uint32(i)
}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID,
Collection: "",
ShardIds: allShards,
})
if err != nil {
t.Fatalf("VolumeEcShardsMount failed: %v", err)
}
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
})
if err != nil {
t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after ECX corruption")
}
}
// scrubEcUntilLocated runs an EC scrub, retrying while the server is still waiting on
// the master for the shard locations a distributed scrub needs.
func scrubEcUntilLocated(t *testing.T, ctx context.Context, grpcClient volume_server_pb.VolumeServerClient, volumeID uint32, mode volume_server_pb.VolumeScrubMode) *volume_server_pb.ScrubEcVolumeResponse {
t.Helper()
deadline := time.Now().Add(60 * time.Second)
for {
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: mode,
})
if err != nil {
t.Fatalf("ScrubEcVolume %s failed: %v", mode, err)
}
waiting := false
for _, d := range resp.GetDetails() {
if strings.Contains(d, "failed to locate shard via master grpc") {
waiting = true
break
}
}
if !waiting {
return resp
}
if time.Now().After(deadline) {
t.Fatalf("master never reported EC shard locations for volume %d: %v", volumeID, resp.GetDetails())
}
time.Sleep(time.Second)
}
}
// With a shard gone, FULL cannot read the intervals that lived on it, while READS
// rebuilds them from parity. Either way the missing shard has to be reported: a
// volume that scrubs clean is a volume nobody repairs.
func TestScrubEcVolumeReadsRecoversMissingShard(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(216)
const missingShard = uint32(0)
httpClient := framework.NewHTTPClient()
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
if _, err := grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
VolumeId: volumeID,
ShardIds: []uint32{missingShard},
}); err != nil {
t.Fatalf("VolumeEcShardsUnmount shard %d failed: %v", missingShard, err)
}
if _, err := grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: volumeID,
ShardIds: []uint32{missingShard},
}); err != nil {
t.Fatalf("VolumeEcShardsDelete shard %d failed: %v", missingShard, err)
}
fullResp := scrubEcUntilLocated(t, ctx, grpcClient, volumeID, volume_server_pb.VolumeScrubMode_FULL)
assertOnlyBrokenShard(t, "FULL", fullResp, missingShard)
if len(fullResp.GetDetails()) == 0 {
t.Fatalf("FULL should report the needles it could not read")
}
readsResp := scrubEcUntilLocated(t, ctx, grpcClient, volumeID, volume_server_pb.VolumeScrubMode_READS)
assertOnlyBrokenShard(t, "READS", readsResp, missingShard)
if len(readsResp.GetDetails()) != 0 {
t.Fatalf("READS should rebuild every needle from parity, got: %v", readsResp.GetDetails())
}
}
func assertOnlyBrokenShard(t *testing.T, mode string, resp *volume_server_pb.ScrubEcVolumeResponse, shardID uint32) {
t.Helper()
infos := resp.GetBrokenShardInfos()
if len(infos) != 1 || infos[0].GetShardId() != shardID {
t.Fatalf("%s reported broken shards %v, want only shard %d (details: %v)", mode, infos, shardID, resp.GetDetails())
}
}
test/volume_server/grpc/scrub_query_test.go
+165 -8
View File
@@ -17,7 +17,7 @@ func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -55,7 +55,7 @@ func TestScrubEcVolumeMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -79,7 +79,7 @@ func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -105,7 +105,7 @@ func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -149,7 +149,7 @@ func TestScrubVolumeAutoSelectAndAllModes(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -210,7 +210,7 @@ func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -333,7 +333,7 @@ func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -389,7 +389,7 @@ func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -433,3 +433,160 @@ func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) {
t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err)
}
}
func TestScrubVolumeMarkBrokenReadonlyHealthyVolume(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(76)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("healthy data"))
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
MarkBrokenVolumesReadonly: true,
})
if err != nil {
t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly on healthy volume failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) != 0 {
t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
}
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus failed: %v", err)
}
if statusResp.GetIsReadOnly() {
t.Fatalf("healthy volume should not be read-only after scrub with MarkBrokenVolumesReadonly")
}
}
func TestScrubVolumeMarkBrokenReadonlyCorruptVolume(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(77)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("test data"))
framework.CorruptIndexFile(t, clusterHarness.BaseDir(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// scrub without the flag: broken volume reported but not marked read-only
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
})
if err != nil {
t.Fatalf("ScrubVolume without flag failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after corruption")
}
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus after scrub without flag failed: %v", err)
}
if statusResp.GetIsReadOnly() {
t.Fatalf("volume should not be read-only when MarkBrokenVolumesReadonly is false")
}
// scrub with the flag: broken volume should now be marked read-only
resp, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
MarkBrokenVolumesReadonly: true,
})
if err != nil {
t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after corruption with flag")
}
found := false
for _, d := range resp.GetDetails() {
if strings.Contains(d, "is now read-only") {
found = true
break
}
}
if !found {
t.Fatalf("expected 'is now read-only' in details, got: %v", resp.GetDetails())
}
statusResp, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus after MarkBrokenVolumesReadonly scrub failed: %v", err)
}
if !statusResp.GetIsReadOnly() {
t.Fatalf("broken volume should be read-only after MarkBrokenVolumesReadonly scrub")
}
}
func TestScrubVolumeMarkBrokenReadonlyInMaintenanceMode(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(78)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("test data"))
framework.CorruptIndexFile(t, clusterHarness.BaseDir(), volumeID)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
framework.EnableMaintenanceMode(t, ctx, grpcClient)
// marking a volume readonly only restricts a server that is already
// read-only, so maintenance mode does not block fencing broken volumes
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
VolumeIds: []uint32{volumeID},
Mode: volume_server_pb.VolumeScrubMode_INDEX,
MarkBrokenVolumesReadonly: true,
})
if err != nil {
t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly in maintenance mode failed: %v", err)
}
if len(resp.GetBrokenVolumeIds()) == 0 {
t.Fatalf("expected broken volume after corruption")
}
statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus after scrub in maintenance mode failed: %v", err)
}
if !statusResp.GetIsReadOnly() {
t.Fatalf("broken volume should be read-only after MarkBrokenVolumesReadonly scrub in maintenance mode")
}
}
test/volume_server/grpc/tail_test.go
+4 -4
View File
@@ -21,7 +21,7 @@ func TestVolumeTailSenderMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -42,7 +42,7 @@ func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -80,7 +80,7 @@ func TestVolumeTailReceiverMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -176,7 +176,7 @@ func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/tiering_remote_test.go
+5 -5
View File
@@ -17,7 +17,7 @@ func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -60,7 +60,7 @@ func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -94,7 +94,7 @@ func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -161,7 +161,7 @@ func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -190,7 +190,7 @@ func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/grpc/vacuum_test.go
+2 -2
View File
@@ -16,7 +16,7 @@ func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -45,7 +45,7 @@ func TestVacuumMaintenanceModeRejections(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/admin_test.go
+22 -12
View File
@@ -16,15 +16,13 @@ func TestAdminStatusAndHealthz(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil)
if err != nil {
t.Fatalf("create status request: %v", err)
}
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
statusResp := framework.DoRequest(t, client, statusReq)
statusBody := framework.ReadAllAndClose(t, statusResp)
@@ -34,8 +32,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
}
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
t.Fatalf("expected echoed request id, got %q", got)
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected server-generated request id in response header")
}
var payload map[string]interface{}
@@ -47,9 +45,21 @@ func TestAdminStatusAndHealthz(t *testing.T) {
t.Fatalf("status payload missing field %q", field)
}
}
diskStatuses, ok := payload["DiskStatuses"].([]interface{})
if !ok || len(diskStatuses) == 0 {
t.Fatalf("status payload expected non-empty DiskStatuses, got %#v", payload["DiskStatuses"])
}
firstDisk, ok := diskStatuses[0].(map[string]interface{})
if !ok {
t.Fatalf("status payload disk status has unexpected shape: %#v", diskStatuses[0])
}
for _, field := range []string{"dir", "all", "used", "free"} {
if _, found := firstDisk[field]; !found {
t.Fatalf("status disk payload missing field %q: %#v", field, firstDisk)
}
}
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
healthResp := framework.DoRequest(t, client, healthReq)
_ = framework.ReadAllAndClose(t, healthResp)
if healthResp.StatusCode != http.StatusOK {
@@ -58,8 +68,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
}
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
t.Fatalf("expected /healthz echoed request id, got %q", got)
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected /healthz server-generated request id in response header")
}
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
@@ -77,7 +87,7 @@ func TestOptionsMethodsByPort(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
cluster := framework.StartVolumeCluster(t, matrix.P2())
client := framework.NewHTTPClient()
adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/"))
@@ -117,7 +127,7 @@ func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
cluster := framework.StartVolumeCluster(t, matrix.P2())
client := framework.NewHTTPClient()
adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")
@@ -154,7 +164,7 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P3())
cluster := framework.StartVolumeCluster(t, matrix.P3())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
@@ -170,7 +180,7 @@ func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
}
profile := matrix.P3()
profile.AccessUI = true
profile.EnableUIAccess = true
cluster := framework.StartSingleVolumeCluster(t, profile)
client := framework.NewHTTPClient()
test/volume_server/http/auth_test.go
+7 -7
View File
@@ -18,7 +18,7 @@ func TestJWTAuthForWriteAndRead(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -81,7 +81,7 @@ func TestJWTAuthRejectsFidMismatch(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -161,7 +161,7 @@ func TestJWTAuthRejectsExpiredTokens(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -209,7 +209,7 @@ func TestJWTAuthViaQueryParamAndCookie(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -249,7 +249,7 @@ func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -299,7 +299,7 @@ func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -345,7 +345,7 @@ func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) {
}
profile := matrix.P3()
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
clusterHarness := framework.StartVolumeCluster(t, profile)
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/chunk_manifest_test.go
+3 -3
View File
@@ -16,7 +16,7 @@ func TestChunkManifestExpansionAndBypass(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -97,7 +97,7 @@ func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -174,7 +174,7 @@ func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/compressed_read_test.go
+1 -1
View File
@@ -43,7 +43,7 @@ func TestCompressedReadAcceptEncodingMatrix(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/headers_static_test.go
+41 -3
View File
@@ -15,7 +15,7 @@ func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -59,12 +59,50 @@ func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
}
}
func TestDownloadDispositionUsesGoBoolParsing(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(97)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
fullFileID := framework.NewFileID(volumeID, 661123, 0x55667789)
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("dl-bool-parse-content"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
parts := strings.SplitN(fullFileID, ",", 2)
if len(parts) != 2 {
t.Fatalf("unexpected file id format: %q", fullFileID)
}
fidOnly := parts[1]
url := fmt.Sprintf("%s/%d/%s/%s?dl=t", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "report.txt")
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("download read expected 200, got %d", resp.StatusCode)
}
contentDisposition := resp.Header.Get("Content-Disposition")
if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") {
t.Fatalf("download disposition with dl=t mismatch: %q", contentDisposition)
}
}
func TestStaticAssetEndpoints(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico"))
@@ -85,7 +123,7 @@ func TestStaticAssetEndpointsOnPublicPort(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
client := framework.NewHTTPClient()
faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico"))
test/volume_server/http/image_transform_test.go
+1 -1
View File
@@ -45,7 +45,7 @@ func TestImageResizeAndCropReadVariants(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/production_features_test.go
+387
View File
@@ -0,0 +1,387 @@
package volume_server_http_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"mime/multipart"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
func TestStatsEndpoints(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// /stats/counter — expect 200 with non-empty body
// Note: Go server guards these with WhiteList which may return 400
counterResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/counter"))
counterBody := framework.ReadAllAndClose(t, counterResp)
if counterResp.StatusCode == http.StatusBadRequest {
t.Logf("/stats/counter returned 400 (whitelist guard), skipping stats checks")
return
}
if counterResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/counter expected 200, got %d, body: %s", counterResp.StatusCode, string(counterBody))
}
if len(counterBody) == 0 {
t.Fatalf("/stats/counter returned empty body")
}
// /stats/memory — expect 200, valid JSON with Version and Memory
memoryResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/memory"))
memoryBody := framework.ReadAllAndClose(t, memoryResp)
if memoryResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/memory expected 200, got %d, body: %s", memoryResp.StatusCode, string(memoryBody))
}
var memoryPayload map[string]any
if err := json.Unmarshal(memoryBody, &memoryPayload); err != nil {
t.Fatalf("/stats/memory response is not valid JSON: %v, body: %s", err, string(memoryBody))
}
if _, ok := memoryPayload["Version"]; !ok {
t.Fatalf("/stats/memory missing Version field")
}
if _, ok := memoryPayload["Memory"]; !ok {
t.Fatalf("/stats/memory missing Memory field")
}
// /stats/disk — expect 200, valid JSON with Version and DiskStatuses
diskResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/disk"))
diskBody := framework.ReadAllAndClose(t, diskResp)
if diskResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/disk expected 200, got %d, body: %s", diskResp.StatusCode, string(diskBody))
}
var diskPayload map[string]any
if err := json.Unmarshal(diskBody, &diskPayload); err != nil {
t.Fatalf("/stats/disk response is not valid JSON: %v, body: %s", err, string(diskBody))
}
if _, ok := diskPayload["Version"]; !ok {
t.Fatalf("/stats/disk missing Version field")
}
if _, ok := diskPayload["DiskStatuses"]; !ok {
t.Fatalf("/stats/disk missing DiskStatuses field")
}
}
func TestStatusPrettyJsonAndCallbackIgnored(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// ?pretty=y — expect indented multi-line JSON
prettyResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?pretty=y"))
prettyBody := framework.ReadAllAndClose(t, prettyResp)
if prettyResp.StatusCode != http.StatusOK {
t.Fatalf("/status?pretty=y expected 200, got %d", prettyResp.StatusCode)
}
lines := strings.Split(strings.TrimSpace(string(prettyBody)), "\n")
if len(lines) < 3 {
t.Fatalf("/status?pretty=y expected multi-line indented JSON, got %d lines: %s", len(lines), string(prettyBody))
}
var prettyPayload map[string]interface{}
if err := json.Unmarshal(prettyBody, &prettyPayload); err != nil {
t.Fatalf("/status?pretty=y is not valid JSON: %v", err)
}
// ?callback=myFunc — must be ignored; response is plain JSON with nosniff.
cbResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?callback=myFunc"))
cbBody := framework.ReadAllAndClose(t, cbResp)
if cbResp.StatusCode != http.StatusOK {
t.Fatalf("/status?callback=myFunc expected 200, got %d", cbResp.StatusCode)
}
if ct := cbResp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
t.Fatalf("/status?callback=myFunc expected Content-Type application/json, got %q", ct)
}
if nosniff := cbResp.Header.Get("X-Content-Type-Options"); nosniff != "nosniff" {
t.Fatalf("/status?callback=myFunc expected X-Content-Type-Options nosniff, got %q", nosniff)
}
if strings.Contains(string(cbBody), "myFunc(") {
t.Fatalf("/status?callback=myFunc must not wrap response in callback; body: %q", string(cbBody))
}
var cbPayload map[string]interface{}
if err := json.Unmarshal(cbBody, &cbPayload); err != nil {
t.Fatalf("/status?callback=myFunc is not valid JSON: %v", err)
}
}
func TestUploadWithCustomTimestamp(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(91)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 910001, 0xAABBCC01)
client := framework.NewHTTPClient()
data := []byte("custom-timestamp-data")
// Upload with ?ts=1700000000
uploadURL := fmt.Sprintf("%s/%s?ts=1700000000", cluster.VolumeAdminURL(), fid)
req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
if err != nil {
t.Fatalf("create upload request: %v", err)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
uploadResp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload with ts expected 201, got %d", uploadResp.StatusCode)
}
// Read back and verify Last-Modified
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read expected 200, got %d", getResp.StatusCode)
}
expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
gotLastModified := getResp.Header.Get("Last-Modified")
if gotLastModified != expectedLastModified {
t.Fatalf("Last-Modified mismatch: got %q, want %q", gotLastModified, expectedLastModified)
}
}
func TestMultipartUploadUsesFormFieldsForTimestampAndTTL(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
// Go's r.FormValue() cannot read multipart text fields after r.MultipartReader()
// consumes the body, so ts/ttl sent as multipart fields only work with the Rust server.
if os.Getenv("VOLUME_SERVER_IMPL") != "rust" {
t.Skip("skipping: multipart form field extraction for ts/ttl is Rust-only")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(94)
const needleID = uint64(940001)
const cookie = uint32(0xAABBCC04)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, needleID, cookie)
payload := []byte("multipart-form-fields-data")
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("ts", "1700000000"); err != nil {
t.Fatalf("write multipart ts field: %v", err)
}
if err := writer.WriteField("ttl", "7d"); err != nil {
t.Fatalf("write multipart ttl field: %v", err)
}
filePart, err := writer.CreateFormFile("file", "multipart.txt")
if err != nil {
t.Fatalf("create multipart file field: %v", err)
}
if _, err := filePart.Write(payload); err != nil {
t.Fatalf("write multipart file payload: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid, &body)
if err != nil {
t.Fatalf("create multipart upload request: %v", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
client := framework.NewHTTPClient()
uploadResp := framework.DoRequest(t, client, req)
uploadBody := framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("multipart upload expected 201, got %d, body: %s", uploadResp.StatusCode, string(uploadBody))
}
readResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("multipart upload read expected 200, got %d", readResp.StatusCode)
}
expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
if got := readResp.Header.Get("Last-Modified"); got != expectedLastModified {
t.Fatalf("multipart upload Last-Modified mismatch: got %q want %q", got, expectedLastModified)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: needleID,
})
if err != nil {
t.Fatalf("VolumeNeedleStatus after multipart upload failed: %v", err)
}
// Go's ReadTTL normalizes via fitTtlCount: 7d → 1w (7 days = 1 week)
if got := statusResp.GetTtl(); got != "1w" {
t.Fatalf("multipart upload TTL mismatch: got %q want %q", got, "1w")
}
}
func TestRequestIdGeneration(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// GET /status WITHOUT setting x-amz-request-id header
req := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status")
resp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("/status expected 200, got %d", resp.StatusCode)
}
reqID := resp.Header.Get("x-amz-request-id")
if reqID == "" {
t.Fatalf("expected auto-generated x-amz-request-id header, got empty")
}
// Go format: "%X%08X" (timestamp hex + 8 random hex), typically 20-24 chars, all hex, no hyphens.
if len(reqID) < 16 {
t.Fatalf("x-amz-request-id too short: %q (len=%d)", reqID, len(reqID))
}
}
func TestS3ResponsePassthroughHeaders(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(92)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 920001, 0xAABBCC02)
client := framework.NewHTTPClient()
data := []byte("passthrough-headers-test-data")
uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
// Read back with S3 passthrough query params
// Test response-content-language which both Go and Rust support
readURL := fmt.Sprintf("%s/%s?response-content-language=fr&response-expires=%s",
cluster.VolumeAdminURL(), fid,
"Thu,+01+Jan+2099+00:00:00+GMT",
)
readResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, readURL))
readBody := framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("read with passthrough expected 200, got %d, body: %s", readResp.StatusCode, string(readBody))
}
if got := readResp.Header.Get("Content-Language"); got != "fr" {
t.Fatalf("Content-Language expected 'fr', got %q", got)
}
if got := readResp.Header.Get("Expires"); got != "Thu, 01 Jan 2099 00:00:00 GMT" {
t.Fatalf("Expires expected 'Thu, 01 Jan 2099 00:00:00 GMT', got %q", got)
}
}
func TestLargeFileWriteAndRead(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(93)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 930001, 0xAABBCC03)
client := framework.NewHTTPClient()
data := bytes.Repeat([]byte("A"), 1024*1024) // 1MB
uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload 1MB expected 201, got %d", uploadResp.StatusCode)
}
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
getBody := framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read 1MB expected 200, got %d", getResp.StatusCode)
}
if len(getBody) != len(data) {
t.Fatalf("read 1MB body length mismatch: got %d, want %d", len(getBody), len(data))
}
if !bytes.Equal(getBody, data) {
t.Fatalf("read 1MB body content mismatch")
}
}
func TestUploadWithContentTypePreservation(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(94)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 940001, 0xAABBCC04)
client := framework.NewHTTPClient()
data := []byte("fake-png-data-for-content-type-test")
// Upload with Content-Type: image/png
uploadURL := fmt.Sprintf("%s/%s", cluster.VolumeAdminURL(), fid)
req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
if err != nil {
t.Fatalf("create upload request: %v", err)
}
req.Header.Set("Content-Type", "image/png")
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
uploadResp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload with image/png expected 201, got %d", uploadResp.StatusCode)
}
// Read back and verify Content-Type is preserved
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read expected 200, got %d", getResp.StatusCode)
}
if got := getResp.Header.Get("Content-Type"); got != "image/png" {
t.Fatalf("Content-Type expected 'image/png', got %q", got)
}
}
test/volume_server/http/public_cors_methods_test.go
+40 -41
View File
@@ -15,7 +15,7 @@ func TestPublicPortReadOnlyMethodBehavior(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -70,7 +70,7 @@ func TestCorsAndUnsupportedMethodBehavior(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -133,7 +133,7 @@ func TestUnsupportedMethodTraceParity(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -168,7 +168,7 @@ func TestUnsupportedMethodPropfindParity(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -207,12 +207,46 @@ func TestUnsupportedMethodPropfindParity(t *testing.T) {
}
}
func TestPublicPortHeadReadParity(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(86)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 124002, 0x04040404)
payload := []byte("public-head-parity-content")
client := framework.NewHTTPClient()
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
headBody := framework.ReadAllAndClose(t, headResp)
if headResp.StatusCode != http.StatusOK {
t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
}
if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
}
if len(headBody) != 0 {
t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
}
}
func TestUnsupportedMethodConnectParity(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -256,7 +290,7 @@ func TestUnsupportedMethodMkcolParity(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -294,38 +328,3 @@ func TestUnsupportedMethodMkcolParity(t *testing.T) {
t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
}
}
func TestPublicPortHeadReadParity(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(86)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 124002, 0x04040404)
payload := []byte("public-head-parity-content")
client := framework.NewHTTPClient()
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
headBody := framework.ReadAllAndClose(t, headResp)
if headResp.StatusCode != http.StatusOK {
t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
}
if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
}
if len(headBody) != 0 {
t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
}
}
test/volume_server/http/range_variants_test.go
+2 -2
View File
@@ -14,7 +14,7 @@ func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -52,7 +52,7 @@ func TestOversizedCombinedRangesAreIgnored(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/read_deleted_test.go
+1 -1
View File
@@ -14,7 +14,7 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/read_path_variants_test.go
+8 -8
View File
@@ -15,7 +15,7 @@ func TestReadPathShapesAndIfModifiedSince(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -85,7 +85,7 @@ func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid"))
@@ -100,7 +100,7 @@ func TestReadWrongCookieReturnsNotFound(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -139,7 +139,7 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -174,11 +174,11 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
precedenceResp := framework.DoRequest(t, client, precedenceReq)
precedenceBody := framework.ReadAllAndClose(t, precedenceResp)
if precedenceResp.StatusCode != http.StatusNotModified {
t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode)
if precedenceResp.StatusCode != http.StatusOK {
t.Fatalf("mismatched If-None-Match overrides If-Modified-Since: expected 200, got %d", precedenceResp.StatusCode)
}
if len(precedenceBody) != 0 {
t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody))
if string(precedenceBody) != string(payload) {
t.Fatalf("mismatched If-None-Match body mismatch: got %q want %q", string(precedenceBody), string(payload))
}
invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
test/volume_server/http/read_write_delete_test.go
+2 -2
View File
@@ -14,7 +14,7 @@ func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
@@ -112,7 +112,7 @@ func TestInvalidReadPathReturnsBadRequest(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle"))
test/volume_server/http/replication_lifecycle_test.go
+102
View File
@@ -0,0 +1,102 @@
package volume_server_http_test
import (
"context"
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)
func TestReplicatedUploadSucceedsImmediatelyAfterAllocate(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(115)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req := &volume_server_pb.AllocateVolumeRequest{
VolumeId: volumeID,
Replication: "001",
Version: uint32(needle.GetCurrentVersion()),
}
if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
t.Fatalf("allocate replicated volume on node0: %v", err)
}
if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
t.Fatalf("allocate replicated volume on node1: %v", err)
}
client := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 881001, 0x0B0C0D0E)
payload := []byte("replicated-upload-after-allocate")
// The master only learns about replica locations through volume-server
// heartbeats, which lag behind the direct AllocateVolume gRPC calls above.
// In production a client obtains its fid from the master assign flow, which
// guarantees the master already knows every replica; this test crafts the
// fid by hand, so the replicated write would otherwise look up the master
// before the second replica is registered and fail with a 500. Wait until
// the master reports both replicas before uploading.
if !waitForMasterReplicaCount(t, client, clusterHarness.MasterURL(), volumeID, 2, 10*time.Second) {
t.Fatalf("master did not report 2 replica locations for volume %d within deadline", volumeID)
}
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("replicated upload expected 201, got %d", uploadResp.StatusCode)
}
replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
var replicaBody []byte
if !waitForHTTPStatus(t, client, replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
replicaBody = framework.ReadAllAndClose(t, resp)
}) {
t.Fatalf("replica did not become readable within deadline")
}
if string(replicaBody) != string(payload) {
t.Fatalf("replica body mismatch: got %q want %q", string(replicaBody), string(payload))
}
}
// waitForMasterReplicaCount polls the master volume lookup until it reports at
// least want locations for volumeID, or the timeout elapses.
func waitForMasterReplicaCount(t testing.TB, client *http.Client, masterURL string, volumeID uint32, want int, timeout time.Duration) bool {
t.Helper()
lookupURL := fmt.Sprintf("%s/dir/lookup?volumeId=%d", masterURL, volumeID)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, lookupURL))
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode == http.StatusOK {
var result struct {
Locations []struct {
Url string `json:"url"`
} `json:"locations"`
}
if err := json.Unmarshal(body, &result); err == nil && len(result.Locations) >= want {
return true
}
}
time.Sleep(200 * time.Millisecond)
}
return false
}
test/volume_server/http/throttling_test.go
+9 -9
View File
@@ -60,7 +60,7 @@ func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -143,7 +143,7 @@ func TestUploadLimitWaitThenProceed(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -226,7 +226,7 @@ func TestUploadLimitTimeoutThenRecovery(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -289,7 +289,7 @@ func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -336,7 +336,7 @@ func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -398,7 +398,7 @@ func TestDownloadLimitTimeoutThenRecovery(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -593,7 +593,7 @@ func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -646,7 +646,7 @@ func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -691,7 +691,7 @@ func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
clusterHarness := framework.StartVolumeCluster(t, matrix.P8())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/write_delete_variants_test.go
+2 -2
View File
@@ -14,7 +14,7 @@ func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
@@ -71,7 +71,7 @@ func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T)
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/http/write_error_variants_test.go
+2 -2
View File
@@ -15,7 +15,7 @@ func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x"))
@@ -38,7 +38,7 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
test/volume_server/loadtest/loadtest_test.go
+629
View File
@@ -0,0 +1,629 @@
package loadtest
import (
"bytes"
"crypto/rand"
"fmt"
"io"
"net/http"
"os"
"sort"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
)
// Run with:
// go test -v -count=1 -timeout 300s -run BenchmarkVolumeServer ./test/volume_server/loadtest/...
// VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 300s -run BenchmarkVolumeServer ./test/volume_server/loadtest/...
//
// Compare results:
// go test -count=1 -timeout 300s -run BenchmarkVolumeServer -bench . ./test/volume_server/loadtest/... | tee go.txt
// VOLUME_SERVER_IMPL=rust go test -count=1 -timeout 300s -run BenchmarkVolumeServer -bench . ./test/volume_server/loadtest/... | tee rust.txt
// Step-by-step payload sizes: 1KB → 4KB → 16KB → 64KB → 256KB → 1MB → 4MB → 8MB
var payloadSteps = []struct {
name string
size int
}{
{"1KB", 1 << 10},
{"4KB", 4 << 10},
{"16KB", 16 << 10},
{"64KB", 64 << 10},
{"256KB", 256 << 10},
{"1MB", 1 << 20},
{"4MB", 4 << 20},
{"8MB", 8 << 20},
}
func implName() string {
if os.Getenv("VOLUME_SERVER_IMPL") == "rust" {
return "rust"
}
return "go"
}
// setupCluster starts a volume cluster and returns the admin URL and cleanup.
func setupCluster(tb testing.TB) (adminURL string, grpcAddr string, cleanup func()) {
tb.Helper()
cluster := framework.StartVolumeCluster(tb, matrix.P1())
return cluster.VolumeAdminURL(), cluster.VolumeGRPCAddress(), cluster.Stop
}
// allocateVolume allocates a volume via gRPC and returns its ID.
func allocateVolume(tb testing.TB, grpcAddr string, volumeID uint32) {
tb.Helper()
conn, client := framework.DialVolumeServer(tb, grpcAddr)
defer conn.Close()
framework.AllocateVolume(tb, client, volumeID, "loadtest")
}
func makePayload(size int) []byte {
data := make([]byte, size)
rand.Read(data)
return data
}
// uploadFile uploads data and returns the file ID used.
func uploadFile(client *http.Client, adminURL string, volumeID uint32, key uint64, cookie uint32, data []byte) error {
fid := framework.NewFileID(volumeID, key, cookie)
url := fmt.Sprintf("%s/%s", adminURL, fid)
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
resp, err := client.Do(req)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("upload %s: status %d", fid, resp.StatusCode)
}
return nil
}
// downloadFile reads a file and discards the body.
func downloadFile(client *http.Client, adminURL string, volumeID uint32, key uint64, cookie uint32) error {
fid := framework.NewFileID(volumeID, key, cookie)
url := fmt.Sprintf("%s/%s", adminURL, fid)
resp, err := client.Get(url)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("download %s: status %d", fid, resp.StatusCode)
}
return nil
}
// deleteFile deletes a file.
func deleteFile(client *http.Client, adminURL string, volumeID uint32, key uint64, cookie uint32) error {
fid := framework.NewFileID(volumeID, key, cookie)
url := fmt.Sprintf("%s/%s", adminURL, fid)
req, err := http.NewRequest(http.MethodDelete, url, nil)
if err != nil {
return err
}
resp, err := client.Do(req)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
return nil
}
// --- Throughput load tests (not Go benchmarks, manual timing for comparison) ---
// TestBenchmarkVolumeServer runs a suite of load tests printing ops/sec and latency.
func TestBenchmarkVolumeServer(t *testing.T) {
if testing.Short() {
t.Skip("skipping load test in short mode")
}
impl := implName()
adminURL, grpcAddr, cleanup := setupCluster(t)
defer cleanup()
const volumeID = uint32(10)
allocateVolume(t, grpcAddr, volumeID)
httpClient := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
MaxIdleConnsPerHost: 128,
MaxConnsPerHost: 128,
},
}
// opsForSize returns fewer ops for larger payloads to keep test time reasonable.
opsForSize := func(size, concurrency int) int {
switch {
case size >= 4<<20:
if concurrency > 1 {
return 64
}
return 30
case size >= 1<<20:
if concurrency > 1 {
return 200
}
return 100
case size >= 64<<10:
if concurrency > 1 {
return 500
}
return 300
default:
if concurrency > 1 {
return 1000
}
return 500
}
}
// Step-by-step upload: 1KB → 4KB → 16KB → 64KB → 256KB → 1MB → 4MB → 8MB
for _, ps := range payloadSteps {
for _, mode := range []struct {
label string
concurrency int
}{
{"seq", 1},
{"c16", 16},
} {
name := fmt.Sprintf("Upload/%s/%s", ps.name, mode.label)
numOps := opsForSize(ps.size, mode.concurrency)
t.Run(fmt.Sprintf("%s/%s", impl, name), func(t *testing.T) {
payload := makePayload(ps.size)
runThroughputTest(t, impl, name, httpClient, adminURL, volumeID,
payload, numOps, mode.concurrency, false, false)
})
}
}
// Step-by-step download: 1KB → 4KB → 16KB → 64KB → 256KB → 1MB → 4MB → 8MB
for _, ps := range payloadSteps {
for _, mode := range []struct {
label string
concurrency int
}{
{"seq", 1},
{"c16", 16},
} {
name := fmt.Sprintf("Download/%s/%s", ps.name, mode.label)
numOps := opsForSize(ps.size, mode.concurrency)
t.Run(fmt.Sprintf("%s/%s", impl, name), func(t *testing.T) {
payload := makePayload(ps.size)
runThroughputTest(t, impl, name, httpClient, adminURL, volumeID,
payload, numOps, mode.concurrency, true, false)
})
}
}
// Mixed read/write at each size
for _, ps := range payloadSteps {
name := fmt.Sprintf("Mixed/%s/c16", ps.name)
numOps := opsForSize(ps.size, 16)
t.Run(fmt.Sprintf("%s/%s", impl, name), func(t *testing.T) {
payload := makePayload(ps.size)
runThroughputTest(t, impl, name, httpClient, adminURL, volumeID,
payload, numOps, 16, false, true)
})
}
// Delete test
t.Run(fmt.Sprintf("%s/Delete/1KB/c16", impl), func(t *testing.T) {
payload := makePayload(1 << 10)
numOps := 1000
baseKey := uint64(900000)
for i := 0; i < numOps; i++ {
if err := uploadFile(httpClient, adminURL, volumeID, baseKey+uint64(i), 1, payload); err != nil {
t.Fatalf("pre-upload for delete %d: %v", i, err)
}
}
var ops atomic.Int64
var totalLatency atomic.Int64
start := time.Now()
var wg sync.WaitGroup
concurrency := 16
opsPerWorker := numOps / concurrency
for w := 0; w < concurrency; w++ {
workerBase := baseKey + uint64(w*opsPerWorker)
wg.Add(1)
go func(wb uint64) {
defer wg.Done()
for i := 0; i < opsPerWorker; i++ {
opStart := time.Now()
deleteFile(httpClient, adminURL, volumeID, wb+uint64(i), 1)
totalLatency.Add(time.Since(opStart).Nanoseconds())
ops.Add(1)
}
}(workerBase)
}
wg.Wait()
elapsed := time.Since(start)
totalOps := ops.Load()
avgLatencyUs := float64(totalLatency.Load()) / float64(totalOps) / 1000.0
opsPerSec := float64(totalOps) / elapsed.Seconds()
t.Logf("RESULT impl=%-4s test=%-22s ops=%-6d errors=%-4d elapsed=%-10s ops/s=%-10.1f avg_lat=%-10.0fus",
impl, "Delete/1KB/c16", totalOps, 0, elapsed.Round(time.Millisecond), opsPerSec, avgLatencyUs)
})
}
// runThroughputTest is the shared core for throughput tests.
// keyOffset separates key ranges so concurrent tests in the same volume don't collide.
// keyCounter provides globally unique key ranges. Starts at 1 because key=0 is invalid.
var keyCounter atomic.Uint64
func init() {
keyCounter.Store(1)
}
func runThroughputTest(
t *testing.T, impl, name string,
httpClient *http.Client, adminURL string, volumeID uint32,
payload []byte, numOps, concurrency int,
isDownload, isMixed bool,
) {
t.Helper()
// Each call gets a unique key range
baseKey := keyCounter.Add(uint64(numOps*2)) - uint64(numOps*2)
// Pre-upload for download / mixed
if isDownload || isMixed {
for i := 0; i < numOps; i++ {
if err := uploadFile(httpClient, adminURL, volumeID, baseKey+uint64(i), 1, payload); err != nil {
t.Fatalf("pre-upload %d: %v", i, err)
}
}
}
uploadBase := baseKey
if !isDownload && !isMixed {
uploadBase = baseKey + uint64(numOps) // fresh range for uploads
}
var ops atomic.Int64
var errors atomic.Int64
var totalLatency atomic.Int64
start := time.Now()
var wg sync.WaitGroup
opsPerWorker := numOps / concurrency
remainder := numOps % concurrency
for w := 0; w < concurrency; w++ {
n := opsPerWorker
if w < remainder {
n++
}
var workerBase uint64
if w < remainder {
workerBase = uploadBase + uint64(w*(opsPerWorker+1))
} else {
workerBase = uploadBase + uint64(remainder*(opsPerWorker+1)) + uint64((w-remainder)*opsPerWorker)
}
wg.Add(1)
go func(wb uint64, count int) {
defer wg.Done()
for i := 0; i < count; i++ {
key := wb + uint64(i)
opStart := time.Now()
var err error
if isMixed {
if i%2 == 0 {
err = uploadFile(httpClient, adminURL, volumeID, key, 1, payload)
} else {
err = downloadFile(httpClient, adminURL, volumeID, key, 1)
}
} else if isDownload {
err = downloadFile(httpClient, adminURL, volumeID, key, 1)
} else {
err = uploadFile(httpClient, adminURL, volumeID, key, 1, payload)
}
totalLatency.Add(time.Since(opStart).Nanoseconds())
ops.Add(1)
if err != nil {
errors.Add(1)
}
}
}(workerBase, n)
}
wg.Wait()
elapsed := time.Since(start)
totalOps := ops.Load()
totalErrs := errors.Load()
avgLatencyUs := float64(totalLatency.Load()) / float64(totalOps) / 1000.0
opsPerSec := float64(totalOps) / elapsed.Seconds()
throughputMBs := opsPerSec * float64(len(payload)) / (1024 * 1024)
t.Logf("RESULT impl=%-4s test=%-22s ops=%-6d errors=%-4d elapsed=%-10s ops/s=%-10.1f avg_lat=%-10.0fus throughput=%.2f MB/s",
impl, name, totalOps, totalErrs, elapsed.Round(time.Millisecond), opsPerSec, avgLatencyUs, throughputMBs)
}
// TestLatencyPercentiles measures p50/p95/p99 latencies for upload and download at each size.
func TestLatencyPercentiles(t *testing.T) {
if testing.Short() {
t.Skip("skipping load test in short mode")
}
impl := implName()
adminURL, grpcAddr, cleanup := setupCluster(t)
defer cleanup()
const volumeID = uint32(20)
allocateVolume(t, grpcAddr, volumeID)
httpClient := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
MaxIdleConnsPerHost: 64,
MaxConnsPerHost: 64,
},
}
latOpsForSize := func(size int) int {
switch {
case size >= 4<<20:
return 30
case size >= 1<<20:
return 100
default:
return 300
}
}
for _, ps := range payloadSteps {
for _, dl := range []struct {
prefix string
isDownload bool
}{
{"Upload", false},
{"Download", true},
} {
name := fmt.Sprintf("%s/%s", dl.prefix, ps.name)
numOps := latOpsForSize(ps.size)
t.Run(fmt.Sprintf("%s/%s", impl, name), func(t *testing.T) {
payload := makePayload(ps.size)
baseKey := keyCounter.Add(uint64(numOps * 2))
if dl.isDownload {
for i := 0; i < numOps; i++ {
if err := uploadFile(httpClient, adminURL, volumeID, baseKey+uint64(i), 2, payload); err != nil {
t.Fatalf("pre-upload: %v", err)
}
}
}
uploadBase := baseKey
if !dl.isDownload {
uploadBase = baseKey + uint64(numOps)
}
latencies := make([]time.Duration, numOps)
for i := 0; i < numOps; i++ {
key := uploadBase + uint64(i)
start := time.Now()
if dl.isDownload {
downloadFile(httpClient, adminURL, volumeID, key, 2)
} else {
uploadFile(httpClient, adminURL, volumeID, key, 2, payload)
}
latencies[i] = time.Since(start)
}
sortDurations(latencies)
p50 := latencies[len(latencies)*50/100]
p95 := latencies[len(latencies)*95/100]
p99 := latencies[len(latencies)*99/100]
min := latencies[0]
max := latencies[len(latencies)-1]
t.Logf("RESULT impl=%-4s test=%-20s n=%-4d min=%-10s p50=%-10s p95=%-10s p99=%-10s max=%-10s",
impl, name, numOps, min.Round(time.Microsecond), p50.Round(time.Microsecond), p95.Round(time.Microsecond), p99.Round(time.Microsecond), max.Round(time.Microsecond))
})
}
}
}
func sortDurations(d []time.Duration) {
sort.Slice(d, func(i, j int) bool { return d[i] < d[j] })
}
// TestSustainedP99 runs high-concurrency load for a sustained period (default 60s,
// override with LOADTEST_DURATION=120s) and reports p50/p95/p99/p999 latencies.
// This reveals tail latency differences that short tests miss (GC pauses, lock contention, etc).
//
// Run:
//
// go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
// VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
// LOADTEST_DURATION=120s VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
func TestSustainedP99(t *testing.T) {
if testing.Short() {
t.Skip("skipping sustained load test in short mode")
}
duration := 60 * time.Second
if d := os.Getenv("LOADTEST_DURATION"); d != "" {
parsed, err := time.ParseDuration(d)
if err == nil && parsed > 0 {
duration = parsed
}
}
impl := implName()
adminURL, grpcAddr, cleanup := setupCluster(t)
defer cleanup()
httpClient := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
MaxIdleConnsPerHost: 128,
MaxConnsPerHost: 128,
},
}
type scenario struct {
name string
size int
concurrency int
isDownload bool
}
scenarios := []scenario{
{"Upload/1KB/c16", 1 << 10, 16, false},
{"Upload/64KB/c16", 64 << 10, 16, false},
{"Download/1KB/c16", 1 << 10, 16, true},
{"Download/64KB/c16", 64 << 10, 16, true},
}
var nextVolID atomic.Uint32
nextVolID.Store(30)
for _, sc := range scenarios {
t.Run(fmt.Sprintf("%s/%s", impl, sc.name), func(t *testing.T) {
// Each scenario gets its own volume to avoid filling up
volumeID := nextVolID.Add(1) - 1
allocateVolume(t, grpcAddr, volumeID)
payload := makePayload(sc.size)
// Pre-upload a pool of files for download tests
poolSize := 500
baseKey := keyCounter.Add(uint64(poolSize*2)) - uint64(poolSize*2)
if sc.isDownload {
t.Logf("Pre-uploading %d files for download test...", poolSize)
for i := 0; i < poolSize; i++ {
if err := uploadFile(httpClient, adminURL, volumeID, baseKey+uint64(i), 3, payload); err != nil {
t.Fatalf("pre-upload %d: %v", i, err)
}
}
}
// Collect latencies from all workers
type latencyBucket struct {
mu sync.Mutex
latencies []time.Duration
}
bucket := &latencyBucket{
latencies: make([]time.Duration, 0, 100000),
}
var totalOps atomic.Int64
var totalErrors atomic.Int64
deadline := time.Now().Add(duration)
start := time.Now()
// For uploads, pre-seed the pool so subsequent writes are overwrites (no volume fill)
if !sc.isDownload {
t.Logf("Pre-seeding %d files for upload overwrite test...", poolSize)
for i := 0; i < poolSize; i++ {
if err := uploadFile(httpClient, adminURL, volumeID, baseKey+uint64(i), 3, payload); err != nil {
t.Fatalf("pre-seed %d: %v", i, err)
}
}
}
var wg sync.WaitGroup
for w := 0; w < sc.concurrency; w++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
localLats := make([]time.Duration, 0, 8192)
var i uint64
for time.Now().Before(deadline) {
// Cycle through the pool to avoid filling up the volume
key := baseKey + uint64(int(i)%poolSize)
opStart := time.Now()
var err error
if sc.isDownload {
err = downloadFile(httpClient, adminURL, volumeID, key, 3)
} else {
err = uploadFile(httpClient, adminURL, volumeID, key, 3, payload)
}
lat := time.Since(opStart)
localLats = append(localLats, lat)
totalOps.Add(1)
if err != nil {
totalErrors.Add(1)
}
i++
// Flush local buffer periodically
if len(localLats) >= 8192 {
bucket.mu.Lock()
bucket.latencies = append(bucket.latencies, localLats...)
bucket.mu.Unlock()
localLats = localLats[:0]
}
}
// Final flush
if len(localLats) > 0 {
bucket.mu.Lock()
bucket.latencies = append(bucket.latencies, localLats...)
bucket.mu.Unlock()
}
}(w)
}
wg.Wait()
elapsed := time.Since(start)
lats := bucket.latencies
n := len(lats)
ops := totalOps.Load()
errs := totalErrors.Load()
opsPerSec := float64(ops) / elapsed.Seconds()
sortDurations(lats)
pct := func(p float64) time.Duration {
idx := int(float64(n) * p / 100.0)
if idx >= n {
idx = n - 1
}
return lats[idx]
}
t.Logf("RESULT impl=%-4s test=%-22s duration=%-6s ops=%-8d errors=%-4d ops/s=%-10.1f",
impl, sc.name, elapsed.Round(time.Second), ops, errs, opsPerSec)
t.Logf(" p50=%-10s p90=%-10s p95=%-10s p99=%-10s p999=%-10s max=%-10s",
pct(50).Round(time.Microsecond),
pct(90).Round(time.Microsecond),
pct(95).Round(time.Microsecond),
pct(99).Round(time.Microsecond),
pct(99.9).Round(time.Microsecond),
lats[n-1].Round(time.Microsecond))
})
}
}
test/volume_server/matrix/config_profiles.go
+1 -1
View File
@@ -12,7 +12,7 @@ type Profile struct {
EnableJWT bool
JWTSigningKey string
JWTReadKey string
AccessUI bool
EnableUIAccess bool
EnableMaintain bool
ConcurrentUploadLimitMB int
test/volume_server/merge/volume_merge_test.go
+454
View File
@@ -0,0 +1,454 @@
package volume_server_merge_test
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// runWeedShell executes a weed shell command by providing commands via stdin with lock/unlock.
// It uses a timeout to prevent hanging if the weed shell process becomes unresponsive.
func runWeedShell(t *testing.T, weedBinary, masterAddr, shellCommand string) (output string, err error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, weedBinary, "shell", "-master="+masterAddr)
// Wrap command in lock/unlock for cluster-wide operations
shellCommands := "lock\n" + shellCommand + "\nunlock\nexit\n"
cmd.Stdin = strings.NewReader(shellCommands)
outputBytes, err := cmd.CombinedOutput()
output = string(outputBytes)
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
t.Logf("weed shell command '%s' timed out after 30s", shellCommand)
} else {
t.Logf("weed shell command '%s' output: %s, error: %v", shellCommand, output, err)
}
}
return output, err
}
// TestVolumeMergeBasic verifies the basic volume.merge workflow using the weed shell command
func TestVolumeMergeBasic(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
// Start a triple cluster with 3 volume servers (needed for merge which allocates to a third location)
cluster := framework.StartTripleVolumeCluster(t, matrix.P1())
// Connect to volume servers to allocate volumes
conn0, volumeClient0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, volumeClient1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(100)
// Allocate volume on only 2 servers (replicas)
// The merge command will allocate on the 3rd server as a temporary location
framework.AllocateVolume(t, volumeClient0, volumeID, "")
framework.AllocateVolume(t, volumeClient1, volumeID, "")
t.Logf("Successfully allocated volume %d on servers 0 and 1 as replicas", volumeID)
// Get weed binary
weedBinary := os.Getenv("WEED_BINARY")
if weedBinary == "" {
var err error
weedBinary, err = framework.FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("failed to find weed binary: %v", err)
}
}
// Execute volume.merge command via weed shell
output, err := runWeedShell(t, weedBinary, cluster.MasterAddress(), fmt.Sprintf("volume.merge -volumeId %d", volumeID))
t.Logf("volume.merge command output:\n%s", output)
if err != nil {
t.Fatalf("volume.merge command failed: %v\noutput: %s", err, output)
}
// Verify the success message in output
if !strings.Contains(output, fmt.Sprintf("merged volume %d", volumeID)) {
t.Fatalf("expected success message in output, got: %s", output)
}
t.Logf("Successfully executed volume.merge command for volume %d", volumeID)
}
// TestVolumeMergeReadonly verifies that volume.merge requires readonly state
func TestVolumeMergeReadonly(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartTripleVolumeCluster(t, matrix.P1())
// Connect to volume servers
conn0, volumeClient0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, volumeClient1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(101)
// Allocate volumes on only 2 servers (the merge will allocate on the 3rd)
framework.AllocateVolume(t, volumeClient0, volumeID, "")
framework.AllocateVolume(t, volumeClient1, volumeID, "")
// Get weed binary
weedBinary := os.Getenv("WEED_BINARY")
if weedBinary == "" {
var err error
weedBinary, err = framework.FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("failed to find weed binary: %v", err)
}
}
// Test 1: Merge while writable (merge command will mark volumes readonly as needed)
output, err := runWeedShell(t, weedBinary, cluster.MasterAddress(), fmt.Sprintf("volume.merge -volumeId %d", volumeID))
if err != nil {
t.Logf("merge on writable volumes failed: %v\noutput: %s", err, output)
t.Fatalf("volume.merge should work on writable volumes (marks them readonly internally)")
}
if !strings.Contains(output, fmt.Sprintf("merged volume %d", volumeID)) {
t.Fatalf("expected success message in output, got: %s", output)
}
// Verify volumes were marked readonly during merge and restored after
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Check that volumes are writable again after merge (were restored)
status0, err := volumeClient0.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status after merge: %v", err)
}
status1, err := volumeClient1.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status from server 1 after merge: %v", err)
}
if status0.GetIsReadOnly() {
t.Fatalf("expected volume to be writable again after merge on server 0")
}
if status1.GetIsReadOnly() {
t.Fatalf("expected volume to be writable again after merge on server 1")
}
t.Logf("Successfully tested merge on writable volumes and writable restoration")
}
// TestVolumeMergeRestore verifies that merge restores writable state for originally-writable replicas
func TestVolumeMergeRestore(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartTripleVolumeCluster(t, matrix.P1())
conn0, volumeClient0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, volumeClient1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(102)
// Allocate volume on only 2 servers (the merge will allocate on the 3rd)
framework.AllocateVolume(t, volumeClient0, volumeID, "")
framework.AllocateVolume(t, volumeClient1, volumeID, "")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Mark both as readonly
_, err := volumeClient0.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly: %v", err)
}
_, err = volumeClient1.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly on server 1: %v", err)
}
// Get weed binary
weedBinary := os.Getenv("WEED_BINARY")
if weedBinary == "" {
var err error
weedBinary, err = framework.FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("failed to find weed binary: %v", err)
}
}
// Execute volume.merge via shell
output, err := runWeedShell(t, weedBinary, cluster.MasterAddress(), fmt.Sprintf("volume.merge -volumeId %d", volumeID))
t.Logf("volume.merge output: %s, error: %v", output, err)
if err != nil {
t.Fatalf("volume.merge failed: %v\noutput: %s", err, output)
}
if !strings.Contains(output, fmt.Sprintf("merged volume %d", volumeID)) {
t.Fatalf("expected success message in output, got: %s", output)
}
// After merge, verify that originally-writable replicas are writable again
// (The merge command should restore writable state for replicas that were writable before readonly)
// Actually both were writable initially, then marked readonly, so both should be restored
// Poll for writable state restoration instead of fixed sleep
maxRetries := 50 // ~5s total with 100ms sleeps
for retries := 0; retries < maxRetries; retries++ {
status0, err := volumeClient0.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err == nil && !status0.GetIsReadOnly() {
// Server 0 is writable, check server 1
status1, err := volumeClient1.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err == nil && !status1.GetIsReadOnly() {
// Both are writable, break out
break
}
}
if retries < maxRetries-1 {
time.Sleep(100 * time.Millisecond)
}
}
status0Final, err := volumeClient0.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get final status for server 0: %v", err)
}
status1Final, err := volumeClient1.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get final status for server 1: %v", err)
}
if status0Final.GetIsReadOnly() {
t.Fatalf("expected volume %d to be writable on server 0 after merge, but it's still readonly", volumeID)
}
if status1Final.GetIsReadOnly() {
t.Fatalf("expected volume %d to be writable on server 1 after merge, but it's still readonly", volumeID)
}
t.Logf("After merge - volume %d on server 0: readonly=%v, server 1: readonly=%v", volumeID, status0Final.GetIsReadOnly(), status1Final.GetIsReadOnly())
t.Logf("Successfully tested merge and restore workflow for volume %d", volumeID)
}
// TestVolumeMergeTailNeedles verifies the volume.merge command with empty volumes
func TestVolumeMergeTailNeedles(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartTripleVolumeCluster(t, matrix.P1())
conn0, volumeClient0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, volumeClient1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(200)
// Allocate empty volumes on only 2 servers (the merge will allocate on the 3rd)
framework.AllocateVolume(t, volumeClient0, volumeID, "")
framework.AllocateVolume(t, volumeClient1, volumeID, "")
// Mark as readonly
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := volumeClient0.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly on server 0: %v", err)
}
_, err = volumeClient1.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly on server 1: %v", err)
}
// Get weed binary
weedBinary := os.Getenv("WEED_BINARY")
if weedBinary == "" {
var err error
weedBinary, err = framework.FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("failed to find weed binary: %v", err)
}
}
// Execute volume.merge command on empty volumes
output, err := runWeedShell(t, weedBinary, cluster.MasterAddress(), fmt.Sprintf("volume.merge -volumeId %d", volumeID))
t.Logf("merge empty volumes - output: %s, error: %v", output, err)
if err != nil {
t.Fatalf("volume.merge failed on empty volumes: %v\noutput: %s", err, output)
}
// Verify merge completed successfully
if !strings.Contains(output, fmt.Sprintf("merged volume %d", volumeID)) {
t.Fatalf("expected success message in output, got: %s", output)
}
t.Logf("Successfully merged empty volumes %d", volumeID)
}
// TestVolumeMergeDivergentReplicas simulates a realistic merge scenario using shell command
func TestVolumeMergeDivergentReplicas(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartTripleVolumeCluster(t, matrix.P1())
// Connect to both servers
conn0, volumeClient0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, volumeClient1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const volumeID = uint32(201)
// Allocate the same volume on only 2 servers (the merge will allocate on the 3rd)
framework.AllocateVolume(t, volumeClient0, volumeID, "")
framework.AllocateVolume(t, volumeClient1, volumeID, "")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Verify both volumes are initially writable
status0, err := volumeClient0.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status for server 0: %v", err)
}
status1, err := volumeClient1.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status for server 1: %v", err)
}
if status0.GetIsReadOnly() || status1.GetIsReadOnly() {
t.Fatalf("expected both volumes to be writable initially")
}
// Mark both as readonly to simulate merge precondition
_, err = volumeClient0.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly on server 0: %v", err)
}
_, err = volumeClient1.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: false,
})
if err != nil {
t.Fatalf("failed to mark readonly on server 1: %v", err)
}
// Verify both are readonly
status0Again, err := volumeClient0.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status after readonly: %v", err)
}
if !status0Again.GetIsReadOnly() {
t.Fatalf("expected volume %d to be readonly", volumeID)
}
// Also verify server 1 is readonly
status1Again, err := volumeClient1.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
VolumeId: volumeID,
})
if err != nil {
t.Fatalf("failed to get status on server 1 after readonly: %v", err)
}
if !status1Again.GetIsReadOnly() {
t.Fatalf("expected volume %d to be readonly on server 1", volumeID)
}
// Get weed binary
weedBinary := os.Getenv("WEED_BINARY")
if weedBinary == "" {
var err error
weedBinary, err = framework.FindOrBuildWeedBinary()
if err != nil {
t.Fatalf("failed to find weed binary: %v", err)
}
}
// Execute volume.merge command via shell
output, err := runWeedShell(t, weedBinary, cluster.MasterAddress(), fmt.Sprintf("volume.merge -volumeId %d", volumeID))
t.Logf("merge divergent replicas - output: %s, error: %v", output, err)
if err != nil {
t.Fatalf("volume.merge failed: %v\noutput: %s", err, output)
}
// Verify merge completed successfully
if !strings.Contains(output, fmt.Sprintf("merged volume %d", volumeID)) {
t.Fatalf("expected success message in output, got: %s", output)
}
t.Logf("Successfully merged divergent replicas for volume %d using shell command", volumeID)
}
test/volume_server/rust/rust_volume_test.go
+310
View File
@@ -0,0 +1,310 @@
package volume_server_rust_test
import (
"context"
"encoding/json"
"net/http"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
func mustNewRequest(t testing.TB, method, url string) *http.Request {
t.Helper()
req, err := http.NewRequest(method, url, nil)
if err != nil {
t.Fatalf("create request %s %s: %v", method, url, err)
}
return req
}
// TestRustHealthzEndpoint verifies that the Rust volume server responds to
// GET /healthz with HTTP 200.
func TestRustHealthzEndpoint(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz"))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected /healthz 200, got %d", resp.StatusCode)
}
}
// TestRustStatusEndpoint verifies that GET /status returns 200 with a JSON
// body containing a "version" field. The Rust server uses lowercase field
// names in its axum JSON responses.
func TestRustStatusEndpoint(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status"))
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected /status 200, got %d, body: %s", resp.StatusCode, string(body))
}
var payload map[string]interface{}
if err := json.Unmarshal(body, &payload); err != nil {
t.Fatalf("decode /status JSON: %v", err)
}
if _, ok := payload["Version"]; !ok {
t.Fatalf("/status JSON missing \"Version\" field, keys: %v", keys(payload))
}
}
// TestRustPingRPC verifies the gRPC Ping RPC returns non-zero timestamps.
func TestRustPingRPC(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
if err != nil {
t.Fatalf("Ping RPC failed: %v", err)
}
if resp.GetStartTimeNs() == 0 {
t.Fatalf("Ping StartTimeNs should be non-zero")
}
if resp.GetStopTimeNs() == 0 {
t.Fatalf("Ping StopTimeNs should be non-zero")
}
if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
t.Fatalf("Ping StopTimeNs (%d) should be >= StartTimeNs (%d)", resp.GetStopTimeNs(), resp.GetStartTimeNs())
}
}
// TestRustAllocateAndWriteReadDelete exercises the full needle lifecycle:
// allocate a volume via gRPC, upload bytes via HTTP POST, read them back
// via HTTP GET, delete via HTTP DELETE, then confirm GET returns 404.
func TestRustAllocateAndWriteReadDelete(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(1)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 1001, 0xAABBCCDD)
data := []byte("rust-volume-server-integration-test-payload")
// Upload
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, data)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
// Read back
getResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
getBody := framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read expected 200, got %d", getResp.StatusCode)
}
if string(getBody) != string(data) {
t.Fatalf("read body mismatch: got %q, want %q", string(getBody), string(data))
}
// Delete
deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid))
_ = framework.ReadAllAndClose(t, deleteResp)
if deleteResp.StatusCode != http.StatusAccepted && deleteResp.StatusCode != http.StatusOK {
t.Fatalf("delete expected 202 or 200, got %d", deleteResp.StatusCode)
}
// Verify 404 after delete
gone := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, gone)
if gone.StatusCode != http.StatusNotFound {
t.Fatalf("read after delete expected 404, got %d", gone.StatusCode)
}
}
// TestRustVolumeLifecycle tests the volume admin gRPC lifecycle:
// allocate, check status, unmount, mount, delete.
func TestRustVolumeLifecycle(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
const volumeID = uint32(2)
framework.AllocateVolume(t, client, volumeID, "")
// VolumeStatus should succeed on a freshly allocated volume.
statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeStatus failed: %v", err)
}
if statusResp.GetFileCount() != 0 {
t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount())
}
// Unmount then remount.
if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
t.Fatalf("VolumeUnmount failed: %v", err)
}
if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
t.Fatalf("VolumeMount failed: %v", err)
}
// Delete.
if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
t.Fatalf("VolumeDelete failed: %v", err)
}
// VolumeStatus should fail after delete.
_, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
if err == nil {
t.Fatalf("VolumeStatus should fail after delete")
}
}
// TestRustGetSetState verifies GetState returns a non-nil state and SetState
// echoes the state back.
func TestRustGetSetState(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// GetState should return non-nil state.
getResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
if err != nil {
t.Fatalf("GetState failed: %v", err)
}
if getResp.GetState() == nil {
t.Fatalf("GetState returned nil state")
}
// SetState should echo back the state.
setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
State: &volume_server_pb.VolumeServerState{
Version: getResp.GetState().GetVersion(),
},
})
if err != nil {
t.Fatalf("SetState failed: %v", err)
}
if setResp.GetState() == nil {
t.Fatalf("SetState returned nil state")
}
if setResp.GetState().GetVersion() < getResp.GetState().GetVersion() {
t.Fatalf("SetState version should not decrease: got %d, had %d",
setResp.GetState().GetVersion(), getResp.GetState().GetVersion())
}
}
// TestRustVolumeServerStatus verifies VolumeServerStatus returns a version
// string and at least one disk status entry.
func TestRustVolumeServerStatus(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
if err != nil {
t.Fatalf("VolumeServerStatus failed: %v", err)
}
if resp.GetVersion() == "" {
t.Fatalf("VolumeServerStatus returned empty version")
}
if len(resp.GetDiskStatuses()) == 0 {
t.Fatalf("VolumeServerStatus returned no disk statuses")
}
}
// TestRustMetricsEndpointIsNotOnAdminPortByDefault verifies that the default
// volume admin listener does not expose Prometheus metrics. Go serves metrics
// only on the dedicated metrics listener when -metricsPort is configured.
func TestRustMetricsEndpointIsNotOnAdminPortByDefault(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartRustVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/metrics"))
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected admin /metrics 400 when metricsPort is unset, got %d body=%s", resp.StatusCode, string(body))
}
}
func TestRustUiAccessOverrideIgnoresReadJwt(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
profile := matrix.P3()
profile.EnableUIAccess = true
cluster := framework.StartRustVolumeCluster(t, profile)
client := framework.NewHTTPClient()
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
body := framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected /ui/index.html 200 with access.ui override, got %d body=%s", resp.StatusCode, string(body))
}
if len(body) == 0 {
t.Fatalf("expected non-empty UI response body")
}
}
// keys returns the keys of a map for diagnostic messages.
func keys(m map[string]interface{}) []string {
ks := make([]string, 0, len(m))
for k := range m {
ks = append(ks, k)
}
return ks
}