seaweedfs/.github/workflows/container_release_unified.yml

name: "docker: build all release containers (unified)"

on:
  push:
    tags:
      - '*'
  workflow_dispatch:
    inputs:
      variant:
        description: 'Variant to build manually'
        required: true
        type: choice
        default: all
        options:
          - all
          - normal
          - large_disk
          - full
          - large_disk_full
          - rocksdb
      release_tag:
        description: 'Release tag to publish (e.g. 3.93)'
        required: true
        default: ''
      rocksdb_version:
        description: 'RocksDB git tag to use when variant=rocksdb'
        required: false
        default: 'v10.10.1'

permissions:
  contents: read
  security-events: write

env:
  RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
  IMAGE: ghcr.io/chrislusf/seaweedfs

# Limit concurrent builds to avoid rate limits
concurrency:
  group: release-${{ github.ref }}
  cancel-in-progress: false

jobs:

  # ── Pre-build the Rust binaries natively ────────────────────────────
  # The volume server and the Rust maintenance worker, cross-compiled for
  # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
  # ~15 minutes of native compilation.
  build-rust-binaries:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        include:
          - target: x86_64-unknown-linux-musl
            arch: amd64
          - target: aarch64-unknown-linux-musl
            arch: arm64
            cross: true
    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}

      - name: Install musl tools (amd64)
        if: ${{ !matrix.cross }}
        run: sudo apt-get install -y musl-tools

      - name: Install cross-compilation tools (arm64)
        if: matrix.cross
        run: |
          sudo apt-get install -y gcc-aarch64-linux-gnu
          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"

      - name: Cache cargo registry and target
        uses: actions/cache@v6
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            seaweed-volume/target
            seaweed-worker/target/${{ matrix.target }}/release
          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
          restore-keys: |
            rust-docker-${{ matrix.target }}-

      # lance's build scripts compile their own protos and look for a protoc.
      # Point them at the one protoc-bin-vendored ships, which seaweed-worker's
      # own build already uses, so no job depends on a system package and every
      # build sees the same version.
      - name: Use the vendored protoc
        run: |
          cd seaweed-worker
          cargo fetch
          # The version from the lock, not whatever else a restored cache holds.
          version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
          test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
          protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
          test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
          echo "PROTOC=$protoc" >> "$GITHUB_ENV"

      - name: Build large-disk variant
        env:
          SEAWEEDFS_COMMIT: ${{ github.sha }}
        run: |
          cd seaweed-volume
          cargo build --release --target ${{ matrix.target }}
          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}

      - name: Build normal variant
        env:
          SEAWEEDFS_COMMIT: ${{ github.sha }}
        run: |
          cd seaweed-volume
          cargo build --release --target ${{ matrix.target }} --no-default-features
          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}

      - name: Build the Rust maintenance worker
        run: |
          cd seaweed-worker
          cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
          cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        with:
          name: rust-bins-${{ matrix.arch }}
          path: |
            weed-volume-large-disk-${{ matrix.arch }}
            weed-volume-normal-${{ matrix.arch }}
            weed-worker-${{ matrix.arch }}

  # One job per (variant, platform) on a native runner, pushed by digest;
  # the merge job stitches the digests into one multi-arch tag.
  build:
    needs: [build-rust-binaries]
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Normal volume - multi-arch
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
          - { variant: normal, tag_suffix: "",             dockerfile: ./docker/Dockerfile.go_build, build_args: "",               rust_variant: normal,     platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }

          # Large disk - multi-arch
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64,  arch: amd64, runner: ubuntu-latest,    qemu: false }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64,  arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest,    qemu: true }
          - { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386,    arch: i386,  runner: ubuntu-latest,    qemu: false }

          # Full tags - multi-arch
          - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
          - { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }

          # Large disk + full tags - multi-arch
          - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest,    qemu: false }
          - { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }

          # RocksDB large disk - amd64 only
          - { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
    steps:
      - name: Skip unselected variant
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
        run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0

      - name: Checkout
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/checkout@v7

      - name: Download pre-built Rust binaries
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/download-artifact@v8
        with:
          pattern: rust-bins-*
          merge-multiple: true
          path: ./rust-bins

      - name: Place Rust binaries in Docker context
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          mkdir -p docker/weed-volume-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
              echo "Placed pre-built Rust binary for ${arch}"
            fi
          done
          mkdir -p docker/weed-worker-prebuilt
          for arch in amd64 arm64; do
            src="./rust-bins/weed-worker-${arch}"
            if [ -f "$src" ]; then
              cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
              echo "Placed pre-built Rust worker for ${arch}"
            fi
          done
          ls -la docker/weed-volume-prebuilt/
          ls -la docker/weed-worker-prebuilt/

      - name: Free Disk Space
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo apt-get clean
          sudo rm -rf /var/lib/apt/lists/*
          sudo docker system prune -af --volumes
          [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
          [ -d /go/pkg ] && rm -rf /go/pkg || true
          df -h

      - name: Docker meta
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        id: docker_meta
        uses: docker/metadata-action@v6
        with:
          images: ${{ env.IMAGE }}
          tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
          flavor: latest=false
          labels: |
            org.opencontainers.image.title=seaweedfs
            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
            org.opencontainers.image.vendor=Chris Lu

      - name: Set up QEMU
        if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
        uses: docker/setup-qemu-action@v4.2.0

      - name: Create BuildKit config
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          cat > /tmp/buildkitd.toml <<EOF
          [registry."docker.io"]
            mirrors = ["https://mirror.gcr.io"]
          EOF

      - name: Set up Docker Buildx
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: docker/setup-buildx-action@v4
        with:
          buildkitd-config: /tmp/buildkitd.toml

      - name: Login to GHCR
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: docker/login-action@v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        id: build
        uses: docker/build-push-action@v7
        env:
          DOCKER_BUILDKIT: 1
        with:
          context: ./docker
          file: ${{ matrix.dockerfile }}
          platforms: ${{ matrix.platform }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          # Flat single-platform manifest so imagetools create assembles cleanly.
          provenance: false
          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
          cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
          # max only for rocksdb: its RocksDB compile is sha-independent and worth
          # keeping; go-build layers are sha-busted every release, so min elsewhere.
          cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
          build-args: |
            ${{ matrix.build_args }}
            BUILDKIT_INLINE_CACHE=1
            BRANCH=${{ github.sha }}
            ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}

      - name: Export digest
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"

      - name: Upload digest
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/upload-artifact@v7
        with:
          name: digest-${{ matrix.variant }}-${{ matrix.arch }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  # Assemble each variant's per-platform digests into one tag, mirror it to
  # Docker Hub, and sign the result on both registries.
  merge:
    needs: [build]
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        include:
          - { variant: normal,          tag_suffix: "" }
          - { variant: large_disk,      tag_suffix: _large_disk }
          - { variant: full,            tag_suffix: _full }
          - { variant: large_disk_full, tag_suffix: _large_disk_full }
          - { variant: rocksdb,         tag_suffix: _large_disk_rocksdb }
    steps:
      - name: Checkout the signing action
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          sparse-checkout: .github/actions
          persist-credentials: false

      - name: Download digests
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: digest-${{ matrix.variant }}-*
          merge-multiple: true
          path: /tmp/digests

      - name: Set up Docker Buildx
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4

      - name: Login to GHCR
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Create multi-arch tag on GHCR
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        id: manifest
        working-directory: /tmp/digests
        run: |
          docker buildx imagetools create \
            -t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
            --metadata-file /tmp/manifest.json \
            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
          docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
          # The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
          digest=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
          echo "digest=${digest}" >> "$GITHUB_OUTPUT"

      - name: Login to Docker Hub
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install crane
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          cd $(mktemp -d)
          curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
          echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833  go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
          tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
          sudo mv crane /usr/local/bin/
          crane version

      - name: Copy ${{ matrix.variant }} to Docker Hub
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
          retry_with_backoff() {
            local max_attempts=5
            local timeout=1
            local attempt=1
            local exit_code=0
            while [ $attempt -le $max_attempts ]; do
              if "$@"; then
                return 0
              else
                exit_code=$?
              fi
              if [ $attempt -lt $max_attempts ]; then
                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
                sleep $timeout
                timeout=$((timeout * 2))
              fi
              attempt=$((attempt + 1))
            done
            echo "Command failed after $max_attempts attempts" >&2
            return $exit_code
          }

          echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
          retry_with_backoff crane copy \
            ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} \
            chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
          echo "Copied ${{ matrix.variant }} to Docker Hub"

      - name: Sign ${{ matrix.variant }}
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: ./.github/actions/sign-image
        with:
          images: >-
            ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
            chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}

  # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
  # Security for visibility, but never blocks the release. Releases (including
  # `latest`) ship regardless — vulnerabilities are tracked, not gated, since
  # we sometimes need to publish through known findings (e.g. unfixed upstream
  # CVE, base-image lag).
  trivy-scan:
    runs-on: ubuntu-latest
    needs: [merge]
    if: github.event_name == 'push'
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - source_suffix: ""
            variant: normal
          - source_suffix: _large_disk
            variant: large_disk
    steps:
      - name: Login to GHCR
        uses: docker/login-action@v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Trivy report (${{ matrix.variant }})
        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          scan-type: image
          # Scan the multi-arch tag on GHCR (already pushed by the build job).
          # Trivy scans the runner's native platform; OS packages are identical
          # across architectures since they all share the same alpine base.
          image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
          scanners: vuln
          vuln-type: os,library
          severity: HIGH,CRITICAL
          ignore-unfixed: true
          limit-severities-for-sarif: true
          format: sarif
          output: trivy-results.sarif
          exit-code: '0'

      - name: Upload Trivy scan results to GitHub Security
        if: always()
        uses: github/codeql-action/upload-sarif@v4.37.9
        with:
          sarif_file: trivy-results.sarif
          category: trivy-${{ matrix.variant }}

  # Point `latest` (and `latest_large_disk`) at the just-released versioned
  # image. crane tag adds an extra tag to an existing manifest — no rebuild,
  # no QEMU, no separate workflow. Replaces the old container_latest.yml
  # rebuild that often failed or lagged behind the release. Independent of
  # trivy-scan: vuln findings are reported but do not block `latest`. The
  # cosign signature is attached to the digest, so `latest` carries it too.
  tag-latest:
    runs-on: ubuntu-latest
    needs: [merge]
    if: github.event_name == 'push'
    strategy:
      matrix:
        include:
          - source_suffix: ""
            latest_tag: latest
          - source_suffix: _large_disk
            latest_tag: latest_large_disk
    steps:
      - name: Login to Docker Hub
        uses: docker/login-action@v4.6.0
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Login to GHCR
        uses: docker/login-action@v4.6.0
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Install crane
        run: |
          cd $(mktemp -d)
          curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
          echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833  go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
          tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
          sudo mv crane /usr/local/bin/
          crane version

      - name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
        run: |
          retry_with_backoff() {
            local max_attempts=5
            local timeout=1
            local attempt=1
            local exit_code=0
            while [ $attempt -le $max_attempts ]; do
              if "$@"; then
                return 0
              else
                exit_code=$?
              fi
              if [ $attempt -lt $max_attempts ]; then
                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
                sleep $timeout
                timeout=$((timeout * 2))
              fi
              attempt=$((attempt + 1))
            done
            echo "Command failed after $max_attempts attempts" >&2
            return $exit_code
          }

          SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
          DST_TAG="${{ matrix.latest_tag }}"

          echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
          retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"

          echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
          retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"

  helm-release:
    runs-on: ubuntu-latest
    needs: [build]
    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
    permissions:
      contents: write
      pages: write
    steps:
      - uses: actions/checkout@v7
      - name: Publish Helm charts
        uses: stefanprodan/helm-gh-pages@v1.7.0
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          charts_dir: k8s/charts
          target_dir: helm
          branch: gh-pages
          helm_version: "3.18.4"