seaweedfs/.github/workflows/s3-tables-tests.yml

name: "S3 Tables Integration Tests"

on:
  pull_request:

permissions:
  contents: read


jobs:
  s3-tables-tests:
    name: S3 Tables Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run S3 Tables Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/table-buckets
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting S3 Tables Tests ==="
          
          # Run S3 Tables integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "S3 Tables integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/table-buckets
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: s3-tables-test-logs
          path: test/s3tables/table-buckets/test-output.log
          retention-days: 3

  iceberg-catalog-tests:
    name: Iceberg Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Run go mod tidy
        run: go mod tidy

      - name: Run Iceberg Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/catalog
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting Iceberg Catalog Tests ==="
          
          # Run Iceberg catalog integration tests using Makefile (handles build)
          make test 2>&1 | tee test-output.log || {
            echo "Iceberg catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: iceberg-catalog-test-logs
          path: test/s3tables/catalog/test-output.log
          retention-days: 3

  trino-iceberg-catalog-tests:
    name: Trino Iceberg Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Trino image
        run: docker pull trinodb/trino:479

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Trino Iceberg Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/catalog_trino
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting Trino Iceberg Catalog Tests ==="
          
          # Run Trino + Iceberg catalog integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "Trino Iceberg catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog_trino
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: trino-iceberg-catalog-test-logs
          path: test/s3tables/catalog_trino/test-output.log
          retention-days: 3

  dremio-iceberg-catalog-tests:
    name: Dremio Iceberg Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Dremio image
        run: docker pull dremio/dremio-oss:25.2.0

      - name: Pre-pull Python image for PyIceberg writer
        run: docker pull python:3.11-slim

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Dremio Iceberg Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/catalog_dremio
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          docker info
          echo "=== Starting Dremio Iceberg Catalog Tests ==="

          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "Dremio Iceberg catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog_dremio
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi

          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker|dremio)" || true
          echo "=== Dremio containers ==="
          docker ps -a --filter "name=seaweed-dremio" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: dremio-iceberg-catalog-test-logs
          path: test/s3tables/catalog_dremio/test-output.log
          retention-days: 3

  doris-iceberg-catalog-tests:
    name: Doris Iceberg Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 35

    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Doris image
        run: docker pull apache/doris:doris-all-in-one-2.1.0

      - name: Pre-pull Python image for PyIceberg writer
        run: docker pull python:3.11-slim

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Doris Iceberg Catalog Integration Tests
        timeout-minutes: 30
        working-directory: test/s3tables/catalog_doris
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          docker info
          echo "=== Starting Doris Iceberg Catalog Tests ==="

          go test -v -timeout 25m . 2>&1 | tee test-output.log || {
            echo "Doris Iceberg catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog_doris
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi

          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker|doris)" || true
          echo "=== Doris containers ==="
          docker ps -a --filter "name=seaweed-doris" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: doris-iceberg-catalog-test-logs
          path: test/s3tables/catalog_doris/test-output.log
          retention-days: 3

  polaris-integration-tests:
    name: Polaris Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Pre-pull Polaris image
        run: docker pull apache/polaris:latest

      - name: Run Polaris Integration Tests
        timeout-minutes: 25
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting Polaris Tests ==="

          go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
            echo "Polaris integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/polaris
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi

          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: polaris-test-logs
          path: test/s3tables/polaris/test-output.log
          retention-days: 3

  spark-iceberg-catalog-tests:
    name: Spark Iceberg Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Spark image
        run: docker pull apache/spark:3.5.1

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Spark Iceberg Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/catalog_spark
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting Spark Iceberg Catalog Tests ==="
          
          # Run Spark + Iceberg catalog integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "Spark Iceberg catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog_spark
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: spark-iceberg-catalog-test-logs
          path: test/s3tables/catalog_spark/test-output.log
          retention-days: 3

  risingwave-catalog-tests:
    name: RisingWave Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull RisingWave image
        run: |
          docker pull risingwavelabs/risingwave:v2.5.0
          docker pull postgres:16-alpine

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run RisingWave Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/catalog_risingwave
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting RisingWave Catalog Tests ==="
          
          # Run RisingWave catalog integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "RisingWave catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/catalog_risingwave
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: risingwave-catalog-test-logs
          path: test/s3tables/catalog_risingwave/test-output.log
          retention-days: 3

  sts-integration-tests:
    name: STS Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Python image
        run: docker pull python:3

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run STS Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/sts_integration
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting STS Integration Tests ==="
          
          # Run STS integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "STS integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/sts_integration
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: sts-integration-test-logs
          path: test/s3tables/sts_integration/test-output.log
          retention-days: 3

  lakekeeper-integration-tests:
    name: Lakekeeper Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Python image
        run: docker pull python:3
        
      - name: Pre-pull LocalStack image (if needed)
        run: docker pull localstack/localstack:latest || true

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Lakekeeper Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/lakekeeper
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          echo "=== Starting Lakekeeper Integration Tests ==="
          
          # Run Lakekeeper integration tests
          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "Lakekeeper integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/lakekeeper
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi
          
          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker)" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: lakekeeper-integration-test-logs
          path: test/s3tables/lakekeeper/test-output.log
          retention-days: 3

  unity-catalog-integration-tests:
    name: Unity Catalog Integration Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 30

    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Set up Docker
        uses: docker/setup-buildx-action@v4

      - name: Pre-pull Unity Catalog image
        run: docker pull unitycatalog/unitycatalog:v0.4.0

      - name: Pre-pull Python image for delta-rs writer
        run: docker pull python:3.11-slim

      - name: Run go mod tidy
        run: go mod tidy

      - name: Install SeaweedFS
        run: |
          go install -buildvcs=false ./weed

      - name: Run Unity Catalog Integration Tests
        timeout-minutes: 25
        working-directory: test/s3tables/unity_catalog
        run: |
          set -x
          set -o pipefail
          echo "=== System Information ==="
          uname -a
          free -h
          df -h
          docker info
          echo "=== Starting Unity Catalog Tests ==="

          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
            echo "Unity Catalog integration tests failed"
            exit 1
          }

      - name: Show test output on failure
        if: failure()
        working-directory: test/s3tables/unity_catalog
        run: |
          echo "=== Test Output ==="
          if [ -f test-output.log ]; then
            tail -200 test-output.log
          fi

          echo "=== Process information ==="
          ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
          echo "=== Unity Catalog containers ==="
          docker ps -a --filter "name=seaweed-unity-catalog" || true

      - name: Upload test logs on failure
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: unity-catalog-integration-test-logs
          path: test/s3tables/unity_catalog/test-output.log
          retention-days: 3

  s3-tables-build-verification:
    name: S3 Tables Build Verification
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Verify S3 Tables Package Builds
        run: |
          set -x
          echo "=== Building S3 Tables package ==="
          go build ./weed/s3api/s3tables || {
            echo "S3 Tables package build failed"
            exit 1
          }
          echo "S3 Tables package built successfully"

      - name: Verify S3 API Integration Builds
        run: |
          set -x
          echo "=== Building S3 API with S3 Tables integration ==="
          go build ./weed/s3api || {
            echo "S3 API build with S3 Tables failed"
            exit 1
          }
          echo "S3 API with S3 Tables integration built successfully"

      - name: Run Go Tests for S3 Tables Package
        run: |
          set -x
          echo "=== Running Go unit tests for S3 Tables ==="
          go test -v -race -timeout 5m ./weed/s3api/s3tables/... || {
            echo "S3 Tables unit tests failed"
            exit 1
          }
          echo "S3 Tables unit tests passed"

      - name: Verify Iceberg Package Builds
        run: |
          set -x
          echo "=== Building Iceberg package ==="
          go build ./weed/s3api/iceberg || {
            echo "Iceberg package build failed"
            exit 1
          }
          echo "Iceberg package built successfully"

      - name: Run Go Tests for Iceberg Package
        run: |
          set -x
          echo "=== Running Go unit tests for Iceberg ==="
          go test -v -race -timeout 5m ./weed/s3api/iceberg/... || {
            echo "Iceberg unit tests failed"
            exit 1
          }
          echo "Iceberg unit tests passed"

  s3-tables-fmt-check:
    name: S3 Tables Format Check
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Check Go Format
        run: |
          set -x
          echo "=== Checking S3 Tables Go format ==="
          unformatted=$(gofmt -l ./weed/s3api/s3tables)
          if [ -n "$unformatted" ]; then
            echo "Go format check failed - files need formatting"
            echo "$unformatted"
            exit 1
          fi
          echo "All S3 Tables files are properly formatted"

      - name: Check S3 Tables Test Format
        run: |
          set -x
          echo "=== Checking S3 Tables test format ==="
          unformatted=$(gofmt -l ./test/s3tables)
          if [ -n "$unformatted" ]; then
            echo "Go format check failed for tests"
            echo "$unformatted"
            exit 1
          fi
          echo "All S3 Tables test files are properly formatted"

      - name: Check Iceberg Format
        run: |
          set -x
          echo "=== Checking Iceberg Go format ==="
          unformatted=$(gofmt -l ./weed/s3api/iceberg)
          if [ -n "$unformatted" ]; then
            echo "Go format check failed for Iceberg - files need formatting"
            echo "$unformatted"
            exit 1
          fi
          echo "All Iceberg files are properly formatted"

  s3-tables-vet:
    name: S3 Tables Go Vet Check
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    
    steps:
      - name: Check out code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
        id: go

      - name: Run Go Vet
        run: |
          set -x
          echo "=== Running go vet on S3 Tables package ==="
          go vet ./weed/s3api/s3tables/... || {
            echo "go vet check failed"
            exit 1
          }
          echo "go vet checks passed"

      - name: Run Go Vet on Tests
        run: |
          set -x
          echo "=== Running go vet on S3 Tables tests ==="
          go vet ./test/s3tables/... || {
            echo "go vet check failed for tests"
            exit 1
          }
          echo "go vet checks passed for tests"

      - name: Run Go Vet on Iceberg
        run: |
          set -x
          echo "=== Running go vet on Iceberg package ==="
          go vet ./weed/s3api/iceberg/... || {
            echo "go vet check failed for Iceberg"
            exit 1
          }
          echo "go vet checks passed for Iceberg"