From 9b94f00325775354cc7f1247358e6b4596656b12 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2026 19:35:54 -0800 Subject: [PATCH] test: mount pinned Iceberg jars into the Spark catalog tests (#12757) * testutil: fetch pinned Maven Central jars with a mirror fallback Co-Authored-By: Claude Opus 5.5 * test: mount pinned Iceberg jars into the Spark catalog containers Each test started a new Spark container that resolved the Iceberg runtime and AWS bundle through Ivy from Maven Central, so a run downloaded them six times and a single refused request left Spark retrying until the package timed out. Co-Authored-By: Claude Opus 5.5 * ci: cache the Iceberg jars for the Spark catalog tests Co-Authored-By: Claude Opus 5.5 * testutil: test the verified jar download Co-Authored-By: Claude Opus 5.5 * test: bound Maven jar downloads by one deadline shared across retries Co-Authored-By: Claude Opus 5.5 * test: give each Maven mirror attempt its own timeout within the download budget Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/s3-tables-tests.yml | 6 ++ test/s3tables/catalog_spark/setup_test.go | 29 +++-- test/testutil/maven.go | 123 ++++++++++++++++++++++ test/testutil/maven_test.go | 92 ++++++++++++++++ 4 files changed, 242 insertions(+), 8 deletions(-) create mode 100644 test/testutil/maven.go create mode 100644 test/testutil/maven_test.go diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index b8bf48bfda..f0dd2c88fc 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -684,6 +684,12 @@ jobs: pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull apache/spark:3.5.1 + - name: Cache Iceberg jars + uses: actions/cache@v6 + with: + path: ~/.cache/seaweedfs-test-jars + key: iceberg-jars-${{ hashFiles('test/s3tables/catalog_spark/setup_test.go') }} + - name: Run go mod tidy run: go mod tidy diff --git a/test/s3tables/catalog_spark/setup_test.go b/test/s3tables/catalog_spark/setup_test.go index 7d64dfd6eb..745e552490 100644 --- a/test/s3tables/catalog_spark/setup_test.go +++ b/test/s3tables/catalog_spark/setup_test.go @@ -187,18 +187,34 @@ catalog.s3.path-style-access = "true" return configDir } +var icebergJars = []struct{ artifact, sha256 string }{ + {"iceberg-spark-runtime-3.5_2.12", "2aec4b78d806cc81d4e50325b351f6349c90de641544e644c6032f000938618b"}, + {"iceberg-aws-bundle", "f4398dcdfefc0b186b899d500651e6e5ac62abe26e25d5cbe28403eaaec24c20"}, +} + +const ( + icebergVersion = "1.5.2" + sparkJarsDir = "/opt/iceberg-jars" + sparkJarsConfig = sparkJarsDir + "/iceberg-spark-runtime-3.5_2.12-" + icebergVersion + ".jar," + + sparkJarsDir + "/iceberg-aws-bundle-" + icebergVersion + ".jar" +) + func (env *TestEnvironment) startSparkContainer(t *testing.T, configDir string) { t.Helper() + mounts := testcontainers.Mounts(testcontainers.BindMount(configDir, "/config")) + for _, jar := range icebergJars { + path := testutil.MavenJar(t, "org.apache.iceberg", jar.artifact, icebergVersion, jar.sha256) + mounts = append(mounts, testcontainers.BindMount(path, testcontainers.ContainerMountTarget(sparkJarsDir+"/"+filepath.Base(path)))) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() req := testcontainers.ContainerRequest{ Image: "apache/spark:3.5.1", ExposedPorts: []string{"4040/tcp"}, - Mounts: testcontainers.Mounts( - testcontainers.BindMount(configDir, "/config"), - ), + Mounts: mounts, Env: map[string]string{ "SPARK_LOCAL_IP": "localhost", }, @@ -266,8 +282,6 @@ import sys spark_home = os.environ.get("SPARK_HOME", "/opt/spark") python_path = os.path.join(spark_home, "python") py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip")) -ivy_dir = "/tmp/ivy" -os.makedirs(ivy_dir, exist_ok=True) os.environ["AWS_REGION"] = "us-west-2" os.environ["AWS_DEFAULT_REGION"] = "us-west-2" os.environ["AWS_ACCESS_KEY_ID"] = "test" @@ -281,8 +295,7 @@ from pyspark.sql import SparkSession spark = (SparkSession.builder .appName("SeaweedFS Iceberg Test") - .config("spark.jars.ivy", ivy_dir) - .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.2,org.apache.iceberg:iceberg-aws-bundle:1.5.2") + .config("spark.jars", "%s") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog") .config("spark.sql.catalog.iceberg.type", "rest") @@ -303,7 +316,7 @@ spark = (SparkSession.builder .getOrCreate()) %s -`, icebergPort, s3Port, sql) +`, sparkJarsConfig, icebergPort, s3Port, sql) code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript}) var output string diff --git a/test/testutil/maven.go b/test/testutil/maven.go new file mode 100644 index 0000000000..943fe07f75 --- /dev/null +++ b/test/testutil/maven.go @@ -0,0 +1,123 @@ +package testutil + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var mavenCentralMirrors = []string{ + "https://repo1.maven.org/maven2", + "https://maven-central.storage-download.googleapis.com/maven2", +} + +var ( + mavenDownloadBudget = 3 * time.Minute + mavenAttemptTimeout = 75 * time.Second +) + +// MavenJar returns a local copy of a Maven Central jar, verified against its +// pinned SHA-256. The jar is kept under the user cache directory so every test +// in a run, and every run that restores that directory, shares one download. +func MavenJar(t testing.TB, group, artifact, version, sha256Hex string) string { + t.Helper() + + cacheRoot, err := os.UserCacheDir() + if err != nil { + cacheRoot = os.TempDir() + } + dir := filepath.Join(cacheRoot, "seaweedfs-test-jars") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create the jar cache directory: %v", err) + } + name := fmt.Sprintf("%s-%s.jar", artifact, version) + path := filepath.Join(dir, name) + if fileSHA256(path) == sha256Hex { + return path + } + + ctx, cancel := context.WithTimeout(context.Background(), mavenDownloadBudget) + defer cancel() + urlPath := fmt.Sprintf("%s/%s/%s/%s", strings.ReplaceAll(group, ".", "/"), artifact, version, name) + var lastErr error + for attempt := 0; attempt < 3 && ctx.Err() == nil; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + case <-time.After(time.Duration(attempt) * 10 * time.Second): + } + } + for _, mirror := range mavenCentralMirrors { + if ctx.Err() != nil { + break + } + attemptCtx, cancelAttempt := context.WithTimeout(ctx, mavenAttemptTimeout) + lastErr = downloadVerified(attemptCtx, mirror+"/"+urlPath, path, sha256Hex) + cancelAttempt() + if lastErr == nil { + return path + } + t.Logf("download %s: %v", name, lastErr) + } + } + t.Fatalf("download %s from Maven Central: %v", name, lastErr) + return "" +} + +func fileSHA256(path string) string { + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "" + } + return hex.EncodeToString(h.Sum(nil)) +} + +func downloadVerified(ctx context.Context, url, path, sha256Hex string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s: %s", url, resp.Status) + } + + tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + h := sha256.New() + _, copyErr := io.Copy(io.MultiWriter(tmp, h), resp.Body) + closeErr := tmp.Close() + if copyErr != nil { + return fmt.Errorf("GET %s: %w", url, copyErr) + } + if closeErr != nil { + return closeErr + } + if got := hex.EncodeToString(h.Sum(nil)); got != sha256Hex { + return fmt.Errorf("GET %s: sha256 %s, want %s", url, got, sha256Hex) + } + if err := os.Chmod(tmp.Name(), 0o644); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/test/testutil/maven_test.go b/test/testutil/maven_test.go new file mode 100644 index 0000000000..147d4e2e66 --- /dev/null +++ b/test/testutil/maven_test.go @@ -0,0 +1,92 @@ +package testutil + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func TestDownloadVerified(t *testing.T) { + body := []byte("jar contents") + sum := sha256.Sum256(body) + want := hex.EncodeToString(sum[:]) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/stall.jar" { + <-r.Context().Done() + return + } + if r.URL.Path != "/a.jar" { + http.NotFound(w, r) + return + } + w.Write(body) + })) + defer srv.Close() + + dir := t.TempDir() + path := filepath.Join(dir, "a.jar") + ctx := context.Background() + + stallCtx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + if err := downloadVerified(stallCtx, srv.URL+"/stall.jar", path, want); err == nil { + t.Fatal("a stalled download should fail at the deadline") + } + + if err := downloadVerified(ctx, srv.URL+"/missing.jar", path, want); err == nil { + t.Fatal("a 404 should fail") + } + if err := downloadVerified(ctx, srv.URL+"/a.jar", path, "00"); err == nil { + t.Fatal("a checksum mismatch should fail") + } + if entries, _ := os.ReadDir(dir); len(entries) != 0 { + t.Fatalf("failed downloads left %d files behind", len(entries)) + } + + if err := downloadVerified(ctx, srv.URL+"/a.jar", path, want); err != nil { + t.Fatalf("download: %v", err) + } + if got := fileSHA256(path); got != want { + t.Fatalf("sha256 %s, want %s", got, want) + } +} + +func TestMavenJarFallsBackPastStalledMirror(t *testing.T) { + body := []byte("jar contents") + sum := sha256.Sum256(body) + want := hex.EncodeToString(sum[:]) + stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + defer stalled.Close() + healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/org/example/demo/1.0/demo-1.0.jar" { + http.NotFound(w, r) + return + } + w.Write(body) + })) + defer healthy.Close() + + cache := t.TempDir() + t.Setenv("XDG_CACHE_HOME", cache) + t.Setenv("HOME", cache) + t.Setenv("LocalAppData", cache) + oldMirrors, oldBudget, oldAttempt := mavenCentralMirrors, mavenDownloadBudget, mavenAttemptTimeout + mavenCentralMirrors = []string{stalled.URL, healthy.URL} + mavenDownloadBudget, mavenAttemptTimeout = 10*time.Second, 200*time.Millisecond + defer func() { + mavenCentralMirrors, mavenDownloadBudget, mavenAttemptTimeout = oldMirrors, oldBudget, oldAttempt + }() + + path := MavenJar(t, "org.example", "demo", "1.0", want) + if got := fileSHA256(path); got != want { + t.Fatalf("sha256 %s, want %s", got, want) + } +}