mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-25 14:30:43 +01:00
test: bound trino container memory + dump logs when it exits before ready (#12044)
* test: dump trino container state and logs when it exits before ready
When the Trino container dies during startup, waitForTrino only
reported the docker exec error ("container is not running"), which
gives no hint whether the JVM was OOM-killed, the image broke, or the
config is bad. Capture docker inspect exit state and the container
logs in the failure message so runner-side flakes are diagnosable
from the test output alone.
* test: cap trino container memory at 4g in catalog tests
Trino's jvm.config sizes the heap at 80% of host RAM. On a loaded
self-hosted runner the JVM attempts to claim most of system memory
at boot and the container is killed ~15s after docker run — the
flake seen in CI today. Bounding the container to 4g gives the JVM
a fixed ~3.2g ceiling (observed usage is ~750mb), immune to host
memory pressure.
* test: keep docker errors in trino container diagnostics
Report docker inspect/logs failures instead of dropping them, and
include the last docker exec output in the container-dead fatal so a
missing container still explains what happened.
* test: detect trino container exit in readiness fallback
When the polling loop times out while the container still appears
alive, the trino --version fallback may itself fail because the
container has since exited. Re-check its output for the
exited-container signature and route through the diagnostic dump.
Generated with [Devin](https://devin.ai)
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test: dump trino container diagnostics on readiness timeout too
The alive-but-never-ready timeout path reported only the last docker
exec error, losing the container state and logs that the dead-container
path already captures.
Generated with [Devin](https://devin.ai)
Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---------
Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
8c1a0a0cc2
commit
1a0485a43d
1 file changed
+34
-8
@@ -466,6 +466,7 @@ func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string)
|
||||
|
||||
cmd := exec.Command("docker", "run", "-d",
|
||||
"--name", containerName,
|
||||
"--memory", "4g",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir),
|
||||
"-v", fmt.Sprintf("%s:/test", env.dataDir),
|
||||
@@ -484,7 +485,7 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
|
||||
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastOutput []byte
|
||||
retryCount := 0
|
||||
containerDead := false
|
||||
for time.Now().Before(deadline) {
|
||||
// Try system catalog query as a readiness check
|
||||
cmd := exec.Command("docker", "exec", containerName,
|
||||
@@ -495,26 +496,51 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
|
||||
return
|
||||
} else {
|
||||
lastOutput = output
|
||||
outputStr := string(output)
|
||||
if strings.Contains(outputStr, "No such container") ||
|
||||
strings.Contains(outputStr, "is not running") {
|
||||
if trinoContainerExited(output) {
|
||||
containerDead = true
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// If we can't connect to system catalog, try to at least connect to Trino server
|
||||
cmd := exec.Command("docker", "exec", containerName, "trino", "--version")
|
||||
if err := cmd.Run(); err == nil {
|
||||
if output, err := exec.Command("docker", "exec", containerName, "trino", "--version").CombinedOutput(); err == nil {
|
||||
// Trino process is running, even if catalog isn't ready yet
|
||||
// Give it a bit more time
|
||||
time.Sleep(5 * time.Second)
|
||||
return
|
||||
} else {
|
||||
lastOutput = output
|
||||
containerDead = containerDead || trinoContainerExited(output)
|
||||
}
|
||||
|
||||
t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s", string(lastOutput))
|
||||
if containerDead {
|
||||
t.Fatalf("Trino container %s exited before becoming ready\nLast output:\n%s\n%s", containerName, string(lastOutput), trinoContainerDiagnostics(containerName))
|
||||
}
|
||||
t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\n%s", string(lastOutput), trinoContainerDiagnostics(containerName))
|
||||
}
|
||||
|
||||
func trinoContainerExited(output []byte) bool {
|
||||
s := string(output)
|
||||
return strings.Contains(s, "No such container") || strings.Contains(s, "is not running")
|
||||
}
|
||||
|
||||
func trinoContainerDiagnostics(containerName string) string {
|
||||
var b strings.Builder
|
||||
if out, err := exec.Command("docker", "inspect", "-f",
|
||||
"status={{.State.Status}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}",
|
||||
containerName).CombinedOutput(); err == nil {
|
||||
fmt.Fprintf(&b, "docker inspect: %s\n", strings.TrimSpace(string(out)))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "docker inspect failed: %v (%s)\n", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
if out, err := exec.Command("docker", "logs", "--tail", "80", containerName).CombinedOutput(); err == nil {
|
||||
fmt.Fprintf(&b, "docker logs:\n%s", string(out))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "docker logs failed: %v (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func runTrinoSQL(t *testing.T, containerName, sql string) string {
|
||||
|
||||
Reference in new issue
Block a user