seaweedfs/.github/workflows/container_latest.yml

name: "docker: build latest container"

# Manual fallback only. On tag push, container_release_unified.yml already
# re-tags the released versioned image as `latest` / `latest_large_disk`,
# so a full rebuild here is unnecessary. Run this manually if you need to
# rebuild `latest` from an arbitrary ref.
on:
  workflow_dispatch:
    inputs:
      source_ref:
        description: 'Git ref to build (branch, tag, or commit SHA)'
        required: true
        default: 'master'
      image_tag:
        description: 'Docker tag to publish (without variant suffix)'
        required: true
        default: 'latest'
      variant:
        description: 'Variant to build manually'
        required: true
        type: choice
        default: all
        options:
          - all
          - standard
          - large_disk
      publish:
        description: 'Publish images and manifests'
        required: true
        type: boolean
        default: false

permissions:
  contents: read
  security-events: write

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      variants: ${{ steps.set-variants.outputs.variants }}
      publish: ${{ steps.set-publish.outputs.publish }}
    steps:
      - name: Select variants for this run
        id: set-variants
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
            variants="[\"${{ github.event.inputs.variant }}\"]"
          else
            variants='["standard","large_disk"]'
          fi
          echo "variants=$variants" >> "$GITHUB_OUTPUT"
      - name: Select publish mode
        id: set-publish
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
          else
            echo "publish=true" >> "$GITHUB_OUTPUT"
          fi

  # ── Pre-build the Rust binaries natively ────────────────────────────
  build-rust-binaries:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        include:
          - target: x86_64-unknown-linux-musl
            arch: amd64
          - target: aarch64-unknown-linux-musl
            arch: arm64
            cross: true
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}

      - name: Install musl tools (amd64)
        if: ${{ !matrix.cross }}
        run: sudo apt-get install -y musl-tools

      - name: Install cross-compilation tools (arm64)
        if: matrix.cross
        run: |
          sudo apt-get install -y gcc-aarch64-linux-gnu
          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"

      - name: Cache cargo registry and target
        uses: actions/cache@v6
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            seaweed-volume/target
            seaweed-worker/target/${{ matrix.target }}/release
          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
          restore-keys: |
            rust-docker-${{ matrix.target }}-

      # lance's build scripts compile their own protos and look for a protoc.
      # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
      # own build already uses, so no job depends on a system package and every
      # build sees the same version.
      - name: Use the vendored protoc
        run: |
          cd seaweed-worker
          cargo fetch
          # The version from the lock, not whatever else a restored cache holds.
          version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
          test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
          protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
          test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
          echo "PROTOC=$protoc" >> "$GITHUB_ENV"

      - name: Build large-disk variant
        env:
          SEAWEEDFS_COMMIT: ${{ github.sha }}
        run: |
          cd seaweed-volume
          cargo build --release --target ${{ matrix.target }}
          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}

      - name: Build normal variant
        env:
          SEAWEEDFS_COMMIT: ${{ github.sha }}
        run: |
          cd seaweed-volume
          cargo build --release --target ${{ matrix.target }} --no-default-features
          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}

      - name: Build the Rust maintenance worker
        run: |
          cd seaweed-worker
          cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
          cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: rust-bins-${{ matrix.arch }}
          path: |
            weed-volume-large-disk-${{ matrix.arch }}
            weed-volume-normal-${{ matrix.arch }}
            weed-worker-${{ matrix.arch }}

  build:
    needs: [setup, build-rust-binaries]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        platform: [amd64, arm64, arm, 386]
        variant: ${{ fromJSON(needs.setup.outputs.variants) }}

    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
      - name: Free Disk Space
        run: |
          echo "Available disk space before cleanup:"
          df -h
          # Remove pre-installed tools
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          # Clean package managers
          sudo apt-get clean
          sudo rm -rf /var/lib/apt/lists/*
          # Clean Docker aggressively
          sudo docker system prune -af --volumes
          # Clean Go cache if it exists
          [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
          [ -d /go/pkg ] && rm -rf /go/pkg || true
          echo "Available disk space after cleanup:"
          df -h
      
      - name: Configure variant
        id: config
        run: |
          if [ "${{ matrix.variant }}" == "large_disk" ]; then
            echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
            echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
            echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
          else
            echo "tag_suffix=" >> $GITHUB_OUTPUT
            echo "build_args=" >> $GITHUB_OUTPUT
            echo "rust_variant=normal" >> $GITHUB_OUTPUT
          fi

      - name: Download pre-built Rust binaries
        uses: actions/download-artifact@v8
        with:
          pattern: rust-bins-*
          merge-multiple: true
          path: ./rust-bins

      - name: Place Rust binaries in Docker context
        run: |
          mkdir -p docker/weed-volume-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
              echo "Placed pre-built Rust binary for ${arch}"
            fi
          done
          mkdir -p docker/weed-worker-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-worker-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
              echo "Placed pre-built Rust worker for ${arch}"
            fi
          done
          ls -la docker/weed-volume-prebuilt/
          ls -la docker/weed-worker-prebuilt/

      - name: Docker meta
        id: docker_meta
        uses: docker/metadata-action@v6
        with:
          images: |
            chrislusf/seaweedfs
            ghcr.io/chrislusf/seaweedfs
          tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
          labels: |
            org.opencontainers.image.title=seaweedfs
            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
            org.opencontainers.image.vendor=Chris Lu
      - name: Set up QEMU
        if: matrix.platform != 'amd64'
        uses: docker/setup-qemu-action@v4.2.0
      - name: Create BuildKit config
        run: |
          cat > /tmp/buildkitd.toml <<EOF
          [registry."docker.io"]
            mirrors = ["https://mirror.gcr.io"]
          EOF
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          buildkitd-flags: "--debug"
          buildkitd-config: /tmp/buildkitd.toml
      - name: Login to Docker Hub
        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4.6.0
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to GHCR
        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Build ${{ matrix.platform }} ${{ matrix.variant }}
        uses: docker/build-push-action@v7
        env:
          DOCKER_BUILDKIT: 1
        with:
          context: ./docker
          push: ${{ needs.setup.outputs.publish == 'true' }}
          file: ./docker/Dockerfile.go_build
          platforms: linux/${{ matrix.platform }}
          # Push to GHCR only during build to avoid Docker Hub rate limits
          tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
          cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
            ${{ steps.config.outputs.build_args }}
      - name: Clean up build artifacts
        if: always()
        run: |
          # Clean up Docker build cache and temporary files
          sudo docker system prune -f
          # Remove Go build cache
          sudo rm -rf /tmp/go-build*

  trivy-scan:
    runs-on: ubuntu-latest
    needs: [setup, build, build-rust-binaries]
    strategy:
      matrix:
        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
    steps:
      - name: Configure variant
        id: config
        run: |
          if [ "${{ matrix.variant }}" == "large_disk" ]; then
            echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
          else
            echo "tag_suffix=" >> $GITHUB_OUTPUT
          fi
      - name: Login to GHCR
        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Checkout for local scan build
        if: needs.setup.outputs.publish != 'true'
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
      - name: Download pre-built Rust binaries for local scan
        if: needs.setup.outputs.publish != 'true'
        uses: actions/download-artifact@v8
        with:
          pattern: rust-bins-*
          merge-multiple: true
          path: ./rust-bins
      - name: Place Rust binaries in Docker context for local scan
        if: needs.setup.outputs.publish != 'true'
        run: |
          rust_variant="normal"
          if [ "${{ matrix.variant }}" == "large_disk" ]; then
            rust_variant="large-disk"
          fi
          mkdir -p docker/weed-volume-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-volume-${rust_variant}-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
              echo "Placed pre-built Rust binary for ${arch}"
            fi
          done
          mkdir -p docker/weed-worker-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-worker-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
              echo "Placed pre-built Rust worker for ${arch}"
            fi
          done
          ls -la docker/weed-volume-prebuilt/
          ls -la docker/weed-worker-prebuilt/
      - name: Create BuildKit config for local scan build
        if: needs.setup.outputs.publish != 'true'
        run: |
          cat > /tmp/buildkitd.toml <<EOF
          [registry."docker.io"]
            mirrors = ["https://mirror.gcr.io"]
          EOF
      - name: Set up Docker Buildx for local scan build
        if: needs.setup.outputs.publish != 'true'
        uses: docker/setup-buildx-action@v4
        with:
          buildkitd-flags: "--debug"
          buildkitd-config: /tmp/buildkitd.toml
      - name: Build local scan image tarball
        if: needs.setup.outputs.publish != 'true'
        uses: docker/build-push-action@v7
        env:
          DOCKER_BUILDKIT: 1
        with:
          context: ./docker
          file: ./docker/Dockerfile.go_build
          platforms: linux/amd64
          outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
            ${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
      - name: Trivy report (published image)
        if: needs.setup.outputs.publish == 'true'
        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: image
          # Scan amd64 only - OS packages are identical across architectures
          # since they all use the same alpine base, so a single-arch scan
          # provides sufficient coverage without multiplying CI time.
          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
          scanners: vuln
          vuln-type: os,library
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          limit-severities-for-sarif: true
          format: sarif
          output: trivy-results.sarif
          exit-code: '0'
      - name: Trivy report (local tarball)
        if: needs.setup.outputs.publish != 'true'
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
          scanners: vuln
          vuln-type: os,library
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          limit-severities-for-sarif: true
          format: sarif
          output: trivy-results.sarif
          exit-code: '0'
      - name: Upload Trivy scan results to GitHub Security
        uses: github/codeql-action/upload-sarif@v4.37.9
        if: always()
        with:
          sarif_file: trivy-results.sarif
      - name: Trivy gate (published image)
        if: needs.setup.outputs.publish == 'true'
        # Gate only on fixable high/critical vulnerabilities. Non-fixable
        # findings are still visible in the SARIF upload above.
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: image
          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
          scanners: vuln
          vuln-type: os,library
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          format: table
          exit-code: '1'
          skip-setup-trivy: true
      - name: Trivy gate (local tarball)
        if: needs.setup.outputs.publish != 'true'
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
          scanners: vuln
          vuln-type: os,library
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          format: table
          exit-code: '1'
          skip-setup-trivy: true

  create-manifest:
    runs-on: ubuntu-latest
    needs: [setup, build, trivy-scan]
    if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
    permissions:
      contents: read
      id-token: write
    strategy:
      matrix:
        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
    steps:
      - name: Checkout the signing action
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - name: Configure variant
        id: config
        run: |
          if [ "${{ matrix.variant }}" == "large_disk" ]; then
            echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
          else
            echo "tag_suffix=" >> $GITHUB_OUTPUT
          fi

      - name: Docker meta
        id: docker_meta
        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
        with:
          images: |
            chrislusf/seaweedfs
            ghcr.io/chrislusf/seaweedfs
          tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
      - name: Login to Docker Hub
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to GHCR
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Install crane
        run: |
          # Install crane for efficient multi-arch image copying
          cd $(mktemp -d)
          curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
          echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833  go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
          tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
          sudo mv crane /usr/local/bin/
          crane version
      - name: Create and push manifest
        id: manifest
        env:
          BASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}
        run: |
          SUFFIX="${{ steps.config.outputs.tag_suffix }}"
          
          # Create manifest on GHCR first (no rate limits)
          echo "Creating GHCR manifest (no rate limits)..."
          docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
            --metadata-file /tmp/manifest.json \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
            ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
          # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
          DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
          echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
          
          # Copy the complete multi-arch image from GHCR to Docker Hub
          # This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
          echo "Copying manifest from GHCR to Docker Hub..."
          
          # Function to retry with exponential backoff for Docker Hub operations
          retry_with_backoff() {
            local max_attempts=5
            local timeout=1
            local attempt=1
            local exit_code=0
            
            while [ $attempt -le $max_attempts ]; do
              if "$@"; then
                return 0
              else
                exit_code=$?
              fi
              
              if [ $attempt -lt $max_attempts ]; then
                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
                sleep $timeout
                timeout=$((timeout * 2))
              fi
              
              attempt=$((attempt + 1))
            done
            
            echo "Command failed after $max_attempts attempts" >&2
            return $exit_code
          }
          
          # Use crane or skopeo to copy, fallback to docker if not available
          if command -v crane &> /dev/null; then
            echo "Using crane to copy..."
            retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs@${DIGEST} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
          elif command -v skopeo &> /dev/null; then
            echo "Using skopeo to copy..."
            retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
          else
            echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
            # Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
            retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
              ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
          fi

      - name: Sign
        uses: ./.github/actions/sign-image
        with:
          images: >-
            ghcr.io/chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
            chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}