name: "S3 Tables Integration Tests" on: pull_request: paths: - 'weed/s3api/**' - 'weed/filer/**' - 'weed/server/**' - 'test/s3tables/**' - 'go.mod' - 'go.sum' - '.github/workflows/s3-tables-tests.yml' permissions: contents: read jobs: s3-tables-tests: name: S3 Tables Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run S3 Tables Integration Tests timeout-minutes: 25 working-directory: test/s3tables/table-buckets run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting S3 Tables Tests ===" # Run S3 Tables integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "S3 Tables integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/table-buckets run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: s3-tables-test-logs path: test/s3tables/table-buckets/test-output.log retention-days: 3 iceberg-catalog-tests: name: Iceberg Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Run go mod tidy run: go mod tidy - name: Run Iceberg Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/catalog run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting Iceberg Catalog Tests ===" # Run Iceberg catalog integration tests using Makefile (handles build) make test 2>&1 | tee test-output.log || { echo "Iceberg catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: iceberg-catalog-test-logs path: test/s3tables/catalog/test-output.log retention-days: 3 trino-iceberg-catalog-tests: name: Trino Iceberg Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull Trino image run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull trinodb/trino:479 - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Trino Iceberg Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/catalog_trino run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting Trino Iceberg Catalog Tests ===" # Run Trino + Iceberg catalog integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "Trino Iceberg catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog_trino run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: trino-iceberg-catalog-test-logs path: test/s3tables/catalog_trino/test-output.log retention-days: 3 dremio-iceberg-catalog-tests: name: Dremio Iceberg Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull images run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull dremio/dremio-oss:25.2.0 pull python:3.11-slim - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Dremio Iceberg Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/catalog_dremio run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h docker info echo "=== Starting Dremio Iceberg Catalog Tests ===" go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "Dremio Iceberg catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog_dremio run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker|dremio)" || true echo "=== Dremio containers ===" docker ps -a --filter "name=seaweed-dremio" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: dremio-iceberg-catalog-test-logs path: test/s3tables/catalog_dremio/test-output.log retention-days: 3 doris-iceberg-catalog-tests: name: Doris Iceberg Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 35 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull images run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull apache/doris:doris-all-in-one-2.1.0 pull python:3.11-slim - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Doris Iceberg Catalog Integration Tests timeout-minutes: 30 working-directory: test/s3tables/catalog_doris run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h docker info echo "=== Starting Doris Iceberg Catalog Tests ===" go test -v -timeout 25m . 2>&1 | tee test-output.log || { echo "Doris Iceberg catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog_doris run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker|doris)" || true echo "=== Doris containers ===" docker ps -a --filter "name=seaweed-doris" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: doris-iceberg-catalog-test-logs path: test/s3tables/catalog_doris/test-output.log retention-days: 3 polaris-integration-tests: name: Polaris Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull Polaris image run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull apache/polaris:latest - name: Run Polaris Integration Tests timeout-minutes: 25 run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting Polaris Tests ===" go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || { echo "Polaris integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/polaris run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: polaris-test-logs path: test/s3tables/polaris/test-output.log retention-days: 3 spark-iceberg-catalog-tests: name: Spark Iceberg Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull Spark image run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull apache/spark:3.5.1 - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Spark Iceberg Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/catalog_spark run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting Spark Iceberg Catalog Tests ===" # Run Spark + Iceberg catalog integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "Spark Iceberg catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog_spark run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: spark-iceberg-catalog-test-logs path: test/s3tables/catalog_spark/test-output.log retention-days: 3 risingwave-catalog-tests: name: RisingWave Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull RisingWave image run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull risingwavelabs/risingwave:v2.5.0 pull postgres:16-alpine - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run RisingWave Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/catalog_risingwave run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting RisingWave Catalog Tests ===" # Run RisingWave catalog integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "RisingWave catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/catalog_risingwave run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: risingwave-catalog-test-logs path: test/s3tables/catalog_risingwave/test-output.log retention-days: 3 sts-integration-tests: name: STS Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Week stamp for image cache key id: week run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT" - name: Restore python:3 image cache id: python-image uses: actions/cache@v6 with: path: /tmp/python3-image.tar key: python3-image-${{ steps.week.outputs.week }} restore-keys: | python3-image- - name: Load or pull python:3 run: | if [ "${{ steps.python-image.outputs.cache-hit }}" = "true" ]; then docker load -i /tmp/python3-image.tar exit 0 fi pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } if pull python:3; then docker save -o /tmp/python3-image.tar python:3 elif [ -f /tmp/python3-image.tar ]; then # Docker Hub unreachable; fall back to last week's cached image docker load -i /tmp/python3-image.tar else exit 1 fi - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run STS Integration Tests timeout-minutes: 25 working-directory: test/s3tables/sts_integration run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting STS Integration Tests ===" # Run STS integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "STS integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/sts_integration run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: sts-integration-test-logs path: test/s3tables/sts_integration/test-output.log retention-days: 3 lakekeeper-integration-tests: name: Lakekeeper Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Lakekeeper Integration Tests timeout-minutes: 25 working-directory: test/s3tables/lakekeeper run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h echo "=== Starting Lakekeeper Integration Tests ===" # Run Lakekeeper integration tests go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "Lakekeeper integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/lakekeeper run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker)" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: lakekeeper-integration-test-logs path: test/s3tables/lakekeeper/test-output.log retention-days: 3 unity-catalog-integration-tests: name: Unity Catalog Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Configure Docker Hub mirror run: | echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker - name: Pre-pull images run: | pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; } pull unitycatalog/unitycatalog:v0.4.0 pull python:3.11-slim - name: Run go mod tidy run: go mod tidy - name: Install SeaweedFS run: | go install -buildvcs=false ./weed - name: Run Unity Catalog Integration Tests timeout-minutes: 25 working-directory: test/s3tables/unity_catalog run: | set -x set -o pipefail echo "=== System Information ===" uname -a free -h df -h docker info echo "=== Starting Unity Catalog Tests ===" go test -v -timeout 20m . 2>&1 | tee test-output.log || { echo "Unity Catalog integration tests failed" exit 1 } - name: Show test output on failure if: failure() working-directory: test/s3tables/unity_catalog run: | echo "=== Test Output ===" if [ -f test-output.log ]; then tail -200 test-output.log fi echo "=== Process information ===" ps aux | grep -E "(weed|test|docker|unitycatalog)" || true echo "=== Unity Catalog containers ===" docker ps -a --filter "name=seaweed-unity-catalog" || true - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v7 with: name: unity-catalog-integration-test-logs path: test/s3tables/unity_catalog/test-output.log retention-days: 3 s3-tables-build-verification: name: S3 Tables Build Verification runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Verify S3 Tables Package Builds run: | set -x echo "=== Building S3 Tables package ===" go build ./weed/s3api/s3tables || { echo "S3 Tables package build failed" exit 1 } echo "S3 Tables package built successfully" - name: Verify S3 API Integration Builds run: | set -x echo "=== Building S3 API with S3 Tables integration ===" go build ./weed/s3api || { echo "S3 API build with S3 Tables failed" exit 1 } echo "S3 API with S3 Tables integration built successfully" - name: Run Go Tests for S3 Tables Package run: | set -x echo "=== Running Go unit tests for S3 Tables ===" go test -v -race -timeout 5m ./weed/s3api/s3tables/... || { echo "S3 Tables unit tests failed" exit 1 } echo "S3 Tables unit tests passed" - name: Verify Iceberg Package Builds run: | set -x echo "=== Building Iceberg package ===" go build ./weed/s3api/iceberg || { echo "Iceberg package build failed" exit 1 } echo "Iceberg package built successfully" - name: Run Go Tests for Iceberg Package run: | set -x echo "=== Running Go unit tests for Iceberg ===" go test -v -race -timeout 5m ./weed/s3api/iceberg/... || { echo "Iceberg unit tests failed" exit 1 } echo "Iceberg unit tests passed" s3-tables-fmt-check: name: S3 Tables Format Check runs-on: ubuntu-22.04 timeout-minutes: 10 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Check Go Format run: | set -x echo "=== Checking S3 Tables Go format ===" unformatted=$(gofmt -l ./weed/s3api/s3tables) if [ -n "$unformatted" ]; then echo "Go format check failed - files need formatting" echo "$unformatted" exit 1 fi echo "All S3 Tables files are properly formatted" - name: Check S3 Tables Test Format run: | set -x echo "=== Checking S3 Tables test format ===" unformatted=$(gofmt -l ./test/s3tables) if [ -n "$unformatted" ]; then echo "Go format check failed for tests" echo "$unformatted" exit 1 fi echo "All S3 Tables test files are properly formatted" - name: Check Iceberg Format run: | set -x echo "=== Checking Iceberg Go format ===" unformatted=$(gofmt -l ./weed/s3api/iceberg) if [ -n "$unformatted" ]; then echo "Go format check failed for Iceberg - files need formatting" echo "$unformatted" exit 1 fi echo "All Iceberg files are properly formatted" s3-tables-vet: name: S3 Tables Go Vet Check runs-on: ubuntu-22.04 timeout-minutes: 10 steps: - name: Check out code uses: actions/checkout@v7 - name: Set up Go uses: actions/setup-go@v6 with: go-version-file: 'go.mod' id: go - name: Run Go Vet run: | set -x echo "=== Running go vet on S3 Tables package ===" go vet ./weed/s3api/s3tables/... || { echo "go vet check failed" exit 1 } echo "go vet checks passed" - name: Run Go Vet on Tests run: | set -x echo "=== Running go vet on S3 Tables tests ===" go vet ./test/s3tables/... || { echo "go vet check failed for tests" exit 1 } echo "go vet checks passed for tests" - name: Run Go Vet on Iceberg run: | set -x echo "=== Running go vet on Iceberg package ===" go vet ./weed/s3api/iceberg/... || { echo "go vet check failed for Iceberg" exit 1 } echo "go vet checks passed for Iceberg"