mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
chrislu
a481a345ac
Better approach than mixing host and container networks. Changes to docker-compose.yml: - Remove 'network_mode: host' from spark-tests container - Add spark-tests to seaweedfs-spark bridge network - Update SEAWEEDFS_FILER_HOST from 'localhost' to 'seaweedfs-filer' - Add depends_on to ensure services are healthy before tests - Update volume publicUrl from 'localhost:8080' to 'seaweedfs-volume:8080' Changes to workflow: - Remove separate build and test steps - Run tests via 'docker compose up spark-tests' - Use --abort-on-container-exit and --exit-code-from for proper exit codes - Simpler: one step instead of two Benefits: ✓ All components use Docker DNS (seaweedfs-master, seaweedfs-volume, seaweedfs-filer) ✓ No host/container network split or DNS resolution issues ✓ Consistent with how other SeaweedFS integration tests work ✓ Tests are fully containerized and reproducible ✓ Volume server accessible via seaweedfs-volume:8080 for all clients ✓ Automatic volume creation works (master can reach volume via gRPC) ✓ Data writes work (Spark can reach volume via Docker network) This matches the architecture of other integration tests and is cleaner.
429 lines
14 KiB
YAML
429 lines
14 KiB
YAML
name: Spark Integration Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [ master, main ]
|
|
paths:
|
|
- 'test/java/spark/**'
|
|
- 'other/java/hdfs2/**'
|
|
- 'other/java/hdfs3/**'
|
|
- 'other/java/client/**'
|
|
- '.github/workflows/spark-integration-tests.yml'
|
|
pull_request:
|
|
branches: [ master, main ]
|
|
paths:
|
|
- 'test/java/spark/**'
|
|
- 'other/java/hdfs2/**'
|
|
- 'other/java/hdfs3/**'
|
|
- 'other/java/client/**'
|
|
- '.github/workflows/spark-integration-tests.yml'
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
checks: write
|
|
pull-requests: write
|
|
|
|
jobs:
|
|
build-deps:
|
|
name: Build SeaweedFS Dependencies
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
java-version: '11'
|
|
distribution: 'temurin'
|
|
cache: maven
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v5
|
|
with:
|
|
go-version: '1.24'
|
|
|
|
- name: Build SeaweedFS binary
|
|
run: |
|
|
echo "Building SeaweedFS binary (statically linked for Alpine)..."
|
|
cd weed
|
|
CGO_ENABLED=0 go build -o ../docker/weed
|
|
cd ../docker
|
|
# Dockerfile.local expects these files in the build context
|
|
ls -la weed filer.toml entrypoint.sh
|
|
file weed
|
|
echo "✓ SeaweedFS binary built and ready for Docker build"
|
|
|
|
- name: Build SeaweedFS Java dependencies
|
|
run: |
|
|
echo "Building Java client (required by HDFS clients)..."
|
|
cd other/java/client
|
|
mvn clean install -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
|
echo "✓ Java client built and installed to local Maven repo"
|
|
cd ../../..
|
|
|
|
echo "Building HDFS2 client (depends on Java client)..."
|
|
cd other/java/hdfs2
|
|
mvn clean install -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
|
echo "✓ HDFS2 client built"
|
|
cd ../../..
|
|
|
|
echo "Building HDFS3 client (depends on Java client)..."
|
|
cd other/java/hdfs3
|
|
mvn clean install -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
|
echo "✓ HDFS3 client built"
|
|
|
|
- name: Prepare artifacts for upload
|
|
run: |
|
|
echo "Preparing artifacts for upload..."
|
|
mkdir -p artifacts/docker
|
|
mkdir -p artifacts/.m2/repository/com
|
|
cp docker/weed artifacts/docker/
|
|
cp -r ~/.m2/repository/com/seaweedfs artifacts/.m2/repository/com/
|
|
echo "✓ Artifacts prepared"
|
|
|
|
- name: Upload build artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: seaweedfs-build
|
|
path: artifacts/
|
|
retention-days: 1
|
|
|
|
spark-tests:
|
|
name: Spark Integration Tests
|
|
runs-on: ubuntu-latest
|
|
needs: build-deps
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
java-version: '11'
|
|
distribution: 'temurin'
|
|
|
|
- name: Download build artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: seaweedfs-build
|
|
path: build-artifacts
|
|
|
|
- name: Restore Maven repository
|
|
run: |
|
|
echo "Restoring Maven artifacts to ~/.m2/repository..."
|
|
mkdir -p ~/.m2/repository/com
|
|
if [ -d "build-artifacts/.m2/repository/com/seaweedfs" ]; then
|
|
cp -r build-artifacts/.m2/repository/com/seaweedfs ~/.m2/repository/com/
|
|
echo "✓ Maven artifacts restored"
|
|
else
|
|
echo "⚠️ Warning: Maven artifacts not found in download"
|
|
ls -la build-artifacts/
|
|
fi
|
|
|
|
- name: Prepare SeaweedFS binary
|
|
run: |
|
|
echo "Copying SeaweedFS binary to docker directory..."
|
|
ls -la build-artifacts/docker/
|
|
cp build-artifacts/docker/weed docker/
|
|
chmod +x docker/weed
|
|
ls -la docker/weed
|
|
file docker/weed
|
|
echo "✓ Binary is ready"
|
|
|
|
- name: Start SeaweedFS services
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "Cleaning up any existing Docker Compose resources..."
|
|
docker compose down -v || true
|
|
|
|
echo "Starting SeaweedFS with Docker Compose..."
|
|
docker compose build --no-cache
|
|
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
|
|
|
echo "Waiting for SeaweedFS filer to be ready..."
|
|
for i in {1..30}; do
|
|
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
|
echo "✓ SeaweedFS filer is ready!"
|
|
break
|
|
fi
|
|
if [ $i -eq 30 ]; then
|
|
echo "✗ SeaweedFS failed to start after 60 seconds"
|
|
echo ""
|
|
echo "=== Container Status ==="
|
|
docker compose ps -a
|
|
echo ""
|
|
echo "=== Master Container Logs ==="
|
|
docker compose logs seaweedfs-master
|
|
echo ""
|
|
echo "=== Volume Container Logs ==="
|
|
docker compose logs seaweedfs-volume
|
|
echo ""
|
|
echo "=== Filer Container Logs ==="
|
|
docker compose logs seaweedfs-filer
|
|
echo ""
|
|
echo "=== Inspecting master container ==="
|
|
docker compose exec -T seaweedfs-master ls -la /usr/bin/weed || echo "Failed to inspect container"
|
|
exit 1
|
|
fi
|
|
echo "Waiting... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
# Verify all services
|
|
echo "Verifying SeaweedFS services..."
|
|
curl -f http://localhost:9333/cluster/status || exit 1
|
|
curl -f http://localhost:8888/ || exit 1
|
|
|
|
# Check volume server registration
|
|
echo "Checking volume server status..."
|
|
curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available"
|
|
echo "✓ Volume server registered, volumes will be created on-demand during tests"
|
|
|
|
- name: Run Spark integration tests in Docker
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "Running Spark integration tests in Docker container..."
|
|
docker compose up --abort-on-container-exit --exit-code-from spark-tests spark-tests
|
|
echo "✓ Tests completed"
|
|
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: spark-test-results
|
|
path: test/java/spark/target/surefire-reports/
|
|
retention-days: 30
|
|
|
|
- name: Publish test report
|
|
if: always()
|
|
uses: dorny/test-reporter@v1
|
|
with:
|
|
name: Spark Test Results
|
|
path: test/java/spark/target/surefire-reports/*.xml
|
|
reporter: java-junit
|
|
fail-on-error: true
|
|
|
|
- name: Display SeaweedFS logs on failure
|
|
if: failure()
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "=== SeaweedFS Master Logs ==="
|
|
docker compose logs seaweedfs-master
|
|
echo ""
|
|
echo "=== SeaweedFS Volume Logs ==="
|
|
docker compose logs seaweedfs-volume
|
|
echo ""
|
|
echo "=== SeaweedFS Filer Logs ==="
|
|
docker compose logs seaweedfs-filer
|
|
echo ""
|
|
echo "=== Volume List (via weed shell) ==="
|
|
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed to list volumes"
|
|
volume.list
|
|
exit
|
|
EOF
|
|
echo ""
|
|
echo "=== Cluster Status ==="
|
|
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
|
|
|
|
- name: Stop SeaweedFS services
|
|
if: always()
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "Stopping SeaweedFS services..."
|
|
docker compose down -v
|
|
echo "✓ Cleanup completed"
|
|
|
|
spark-example:
|
|
name: Run Spark Example Application
|
|
runs-on: ubuntu-latest
|
|
needs: [build-deps, spark-tests]
|
|
timeout-minutes: 20
|
|
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
java-version: '11'
|
|
distribution: 'temurin'
|
|
|
|
- name: Download build artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: seaweedfs-build
|
|
path: build-artifacts
|
|
|
|
- name: Restore Maven repository
|
|
run: |
|
|
echo "Restoring Maven artifacts to ~/.m2/repository..."
|
|
mkdir -p ~/.m2/repository/com
|
|
if [ -d "build-artifacts/.m2/repository/com/seaweedfs" ]; then
|
|
cp -r build-artifacts/.m2/repository/com/seaweedfs ~/.m2/repository/com/
|
|
echo "✓ Maven artifacts restored"
|
|
else
|
|
echo "⚠️ Warning: Maven artifacts not found in download"
|
|
ls -la build-artifacts/
|
|
fi
|
|
|
|
- name: Prepare SeaweedFS binary
|
|
run: |
|
|
echo "Copying SeaweedFS binary to docker directory..."
|
|
ls -la build-artifacts/docker/
|
|
cp build-artifacts/docker/weed docker/
|
|
chmod +x docker/weed
|
|
ls -la docker/weed
|
|
file docker/weed
|
|
echo "✓ Binary is ready"
|
|
|
|
- name: Cache Apache Spark
|
|
id: cache-spark
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: spark-3.5.0-bin-hadoop3
|
|
key: spark-3.5.0-hadoop3
|
|
|
|
- name: Download Apache Spark
|
|
if: steps.cache-spark.outputs.cache-hit != 'true'
|
|
run: |
|
|
echo "Downloading Apache Spark 3.5.0..."
|
|
wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz
|
|
tar xzf spark-3.5.0-bin-hadoop3.tgz
|
|
echo "✓ Spark downloaded and extracted"
|
|
|
|
- name: Set Spark environment
|
|
run: |
|
|
echo "SPARK_HOME=$(pwd)/spark-3.5.0-bin-hadoop3" >> $GITHUB_ENV
|
|
echo "$(pwd)/spark-3.5.0-bin-hadoop3/bin" >> $GITHUB_PATH
|
|
|
|
- name: Start SeaweedFS services
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "Cleaning up any existing Docker Compose resources..."
|
|
docker compose down -v || true
|
|
|
|
echo "Starting SeaweedFS with Docker Compose..."
|
|
docker compose build --no-cache
|
|
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
|
|
|
echo "Waiting for SeaweedFS filer to be ready..."
|
|
for i in {1..30}; do
|
|
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
|
echo "✓ SeaweedFS filer is ready!"
|
|
break
|
|
fi
|
|
if [ $i -eq 30 ]; then
|
|
echo "✗ SeaweedFS failed to start after 60 seconds"
|
|
echo ""
|
|
echo "=== Container Status ==="
|
|
docker compose ps -a
|
|
echo ""
|
|
echo "=== Master Container Logs ==="
|
|
docker compose logs seaweedfs-master
|
|
echo ""
|
|
echo "=== Volume Container Logs ==="
|
|
docker compose logs seaweedfs-volume
|
|
echo ""
|
|
echo "=== Filer Container Logs ==="
|
|
docker compose logs seaweedfs-filer
|
|
exit 1
|
|
fi
|
|
echo "Waiting... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
# Verify all services
|
|
echo "Verifying SeaweedFS services..."
|
|
curl -f http://localhost:9333/cluster/status || exit 1
|
|
curl -f http://localhost:8888/ || exit 1
|
|
|
|
# Check volume server registration
|
|
echo "Checking volume server status..."
|
|
curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available"
|
|
echo "✓ Volume server registered, volumes will be created on-demand during tests"
|
|
|
|
- name: Build project
|
|
working-directory: test/java/spark
|
|
run: mvn clean package -DskipTests
|
|
|
|
- name: Run Spark example application
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "Running Spark example application..."
|
|
spark-submit \
|
|
--class seaweed.spark.SparkSeaweedFSExample \
|
|
--master local[2] \
|
|
--conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
|
|
--conf spark.hadoop.fs.seaweed.filer.host=localhost \
|
|
--conf spark.hadoop.fs.seaweed.filer.port=8888 \
|
|
--conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \
|
|
--conf spark.hadoop.fs.seaweed.replication="" \
|
|
target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \
|
|
seaweedfs://localhost:8888/ci-spark-output
|
|
echo "✓ Example application completed successfully"
|
|
|
|
- name: Verify output
|
|
run: |
|
|
echo "Verifying Spark output in SeaweedFS..."
|
|
curl -s http://localhost:8888/ci-spark-output/ || echo "Output directory listing not available"
|
|
|
|
- name: Display SeaweedFS diagnostics on failure
|
|
if: failure()
|
|
working-directory: test/java/spark
|
|
run: |
|
|
echo "=== SeaweedFS Master Logs ==="
|
|
docker compose logs seaweedfs-master
|
|
echo ""
|
|
echo "=== SeaweedFS Volume Logs ==="
|
|
docker compose logs seaweedfs-volume
|
|
echo ""
|
|
echo "=== SeaweedFS Filer Logs ==="
|
|
docker compose logs seaweedfs-filer
|
|
echo ""
|
|
echo "=== Volume List (via weed shell) ==="
|
|
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed to list volumes"
|
|
volume.list
|
|
exit
|
|
EOF
|
|
echo ""
|
|
echo "=== Cluster Status ==="
|
|
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
|
|
|
|
- name: Stop SeaweedFS services
|
|
if: always()
|
|
working-directory: test/java/spark
|
|
run: docker compose down -v
|
|
|
|
summary:
|
|
name: Test Summary
|
|
runs-on: ubuntu-latest
|
|
needs: [spark-tests, spark-example]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Check test results
|
|
run: |
|
|
echo "=== Spark Integration Tests Summary ==="
|
|
echo "Spark Tests: ${{ needs.spark-tests.result }}"
|
|
echo "Spark Example: ${{ needs.spark-example.result }}"
|
|
|
|
if [ "${{ needs.spark-tests.result }}" != "success" ]; then
|
|
echo "❌ Spark integration tests failed"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "${{ needs.spark-example.result }}" != "success" ] && [ "${{ needs.spark-example.result }}" != "skipped" ]; then
|
|
echo "⚠️ Spark example application had issues"
|
|
fi
|
|
|
|
echo "✅ All critical tests passed"
|
|
|