Files
seaweedfs/test/testutil/maven.go
T
Chris LuandClaude Opus 5.5 9b94f00325 test: mount pinned Iceberg jars into the Spark catalog tests (#12757)
* testutil: fetch pinned Maven Central jars with a mirror fallback

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: mount pinned Iceberg jars into the Spark catalog containers

Each test started a new Spark container that resolved the Iceberg runtime
and AWS bundle through Ivy from Maven Central, so a run downloaded them six
times and a single refused request left Spark retrying until the package
timed out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ci: cache the Iceberg jars for the Spark catalog tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* testutil: test the verified jar download

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: bound Maven jar downloads by one deadline shared across retries

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: give each Maven mirror attempt its own timeout within the download budget

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 19:35:54 -08:00

124 lines
3.1 KiB
Go

package testutil
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
var mavenCentralMirrors = []string{
"https://repo1.maven.org/maven2",
"https://maven-central.storage-download.googleapis.com/maven2",
}
var (
mavenDownloadBudget = 3 * time.Minute
mavenAttemptTimeout = 75 * time.Second
)
// MavenJar returns a local copy of a Maven Central jar, verified against its
// pinned SHA-256. The jar is kept under the user cache directory so every test
// in a run, and every run that restores that directory, shares one download.
func MavenJar(t testing.TB, group, artifact, version, sha256Hex string) string {
t.Helper()
cacheRoot, err := os.UserCacheDir()
if err != nil {
cacheRoot = os.TempDir()
}
dir := filepath.Join(cacheRoot, "seaweedfs-test-jars")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create the jar cache directory: %v", err)
}
name := fmt.Sprintf("%s-%s.jar", artifact, version)
path := filepath.Join(dir, name)
if fileSHA256(path) == sha256Hex {
return path
}
ctx, cancel := context.WithTimeout(context.Background(), mavenDownloadBudget)
defer cancel()
urlPath := fmt.Sprintf("%s/%s/%s/%s", strings.ReplaceAll(group, ".", "/"), artifact, version, name)
var lastErr error
for attempt := 0; attempt < 3 && ctx.Err() == nil; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
case <-time.After(time.Duration(attempt) * 10 * time.Second):
}
}
for _, mirror := range mavenCentralMirrors {
if ctx.Err() != nil {
break
}
attemptCtx, cancelAttempt := context.WithTimeout(ctx, mavenAttemptTimeout)
lastErr = downloadVerified(attemptCtx, mirror+"/"+urlPath, path, sha256Hex)
cancelAttempt()
if lastErr == nil {
return path
}
t.Logf("download %s: %v", name, lastErr)
}
}
t.Fatalf("download %s from Maven Central: %v", name, lastErr)
return ""
}
func fileSHA256(path string) string {
f, err := os.Open(path)
if err != nil {
return ""
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return ""
}
return hex.EncodeToString(h.Sum(nil))
}
func downloadVerified(ctx context.Context, url, path, sha256Hex string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("GET %s: %s", url, resp.Status)
}
tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
h := sha256.New()
_, copyErr := io.Copy(io.MultiWriter(tmp, h), resp.Body)
closeErr := tmp.Close()
if copyErr != nil {
return fmt.Errorf("GET %s: %w", url, copyErr)
}
if closeErr != nil {
return closeErr
}
if got := hex.EncodeToString(h.Sum(nil)); got != sha256Hex {
return fmt.Errorf("GET %s: sha256 %s, want %s", url, got, sha256Hex)
}
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}