From 1a0485a43d9b9a233d6770ea043e2e062f48a1da Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 20:00:06 -0700 Subject: [PATCH] test: bound trino container memory + dump logs when it exits before ready (#12044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: dump trino container state and logs when it exits before ready When the Trino container dies during startup, waitForTrino only reported the docker exec error ("container is not running"), which gives no hint whether the JVM was OOM-killed, the image broke, or the config is bad. Capture docker inspect exit state and the container logs in the failure message so runner-side flakes are diagnosable from the test output alone. * test: cap trino container memory at 4g in catalog tests Trino's jvm.config sizes the heap at 80% of host RAM. On a loaded self-hosted runner the JVM attempts to claim most of system memory at boot and the container is killed ~15s after docker run — the flake seen in CI today. Bounding the container to 4g gives the JVM a fixed ~3.2g ceiling (observed usage is ~750mb), immune to host memory pressure. * test: keep docker errors in trino container diagnostics Report docker inspect/logs failures instead of dropping them, and include the last docker exec output in the container-dead fatal so a missing container still explains what happened. * test: detect trino container exit in readiness fallback When the polling loop times out while the container still appears alive, the trino --version fallback may itself fail because the container has since exited. Re-check its output for the exited-container signature and route through the diagnostic dump. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: dump trino container diagnostics on readiness timeout too The alive-but-never-ready timeout path reported only the last docker exec error, losing the container state and logs that the dead-container path already captures. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../catalog_trino/trino_catalog_test.go | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/test/s3tables/catalog_trino/trino_catalog_test.go b/test/s3tables/catalog_trino/trino_catalog_test.go index ddfc60fdeb..8a50318069 100644 --- a/test/s3tables/catalog_trino/trino_catalog_test.go +++ b/test/s3tables/catalog_trino/trino_catalog_test.go @@ -466,6 +466,7 @@ func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) cmd := exec.Command("docker", "run", "-d", "--name", containerName, + "--memory", "4g", "--add-host", "host.docker.internal:host-gateway", "-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir), "-v", fmt.Sprintf("%s:/test", env.dataDir), @@ -484,7 +485,7 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) { deadline := time.Now().Add(timeout) var lastOutput []byte - retryCount := 0 + containerDead := false for time.Now().Before(deadline) { // Try system catalog query as a readiness check cmd := exec.Command("docker", "exec", containerName, @@ -495,26 +496,51 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) { return } else { lastOutput = output - outputStr := string(output) - if strings.Contains(outputStr, "No such container") || - strings.Contains(outputStr, "is not running") { + if trinoContainerExited(output) { + containerDead = true break } - retryCount++ } time.Sleep(1 * time.Second) } // If we can't connect to system catalog, try to at least connect to Trino server - cmd := exec.Command("docker", "exec", containerName, "trino", "--version") - if err := cmd.Run(); err == nil { + if output, err := exec.Command("docker", "exec", containerName, "trino", "--version").CombinedOutput(); err == nil { // Trino process is running, even if catalog isn't ready yet // Give it a bit more time time.Sleep(5 * time.Second) return + } else { + lastOutput = output + containerDead = containerDead || trinoContainerExited(output) } - t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s", string(lastOutput)) + if containerDead { + t.Fatalf("Trino container %s exited before becoming ready\nLast output:\n%s\n%s", containerName, string(lastOutput), trinoContainerDiagnostics(containerName)) + } + t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\n%s", string(lastOutput), trinoContainerDiagnostics(containerName)) +} + +func trinoContainerExited(output []byte) bool { + s := string(output) + return strings.Contains(s, "No such container") || strings.Contains(s, "is not running") +} + +func trinoContainerDiagnostics(containerName string) string { + var b strings.Builder + if out, err := exec.Command("docker", "inspect", "-f", + "status={{.State.Status}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}", + containerName).CombinedOutput(); err == nil { + fmt.Fprintf(&b, "docker inspect: %s\n", strings.TrimSpace(string(out))) + } else { + fmt.Fprintf(&b, "docker inspect failed: %v (%s)\n", err, strings.TrimSpace(string(out))) + } + if out, err := exec.Command("docker", "logs", "--tail", "80", containerName).CombinedOutput(); err == nil { + fmt.Fprintf(&b, "docker logs:\n%s", string(out)) + } else { + fmt.Fprintf(&b, "docker logs failed: %v (%s)", err, strings.TrimSpace(string(out))) + } + return b.String() } func runTrinoSQL(t *testing.T, containerName, sql string) string {