mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-22 04:07:08 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc97217591 | ||
|
|
7664bb3b7f | ||
|
|
4ae402de0d | ||
|
|
ef72d3976e | ||
|
|
153ab668c9 | ||
|
|
6dd6ad512c | ||
|
|
515380c349 | ||
|
|
9620e7ebfe | ||
|
|
53834117a6 | ||
|
|
88b91cccba | ||
|
|
fefac028ec | ||
|
|
0c9872f842 | ||
|
|
da1d7444b8 | ||
|
|
d15396fc4d | ||
|
|
39bdbc18db | ||
|
|
403d51631b | ||
|
|
52d91008ac |
No files matched your search
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
e2e:
|
||||
name: FUSE Mount
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
ec-integration-tests:
|
||||
name: EC Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -33,7 +33,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 50
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
jobs:
|
||||
metadata-subscribe-integration:
|
||||
name: Metadata Subscribe Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -27,7 +27,7 @@ permissions:
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
@@ -270,7 +270,7 @@ jobs:
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -37,9 +37,13 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
@@ -120,7 +124,7 @@ jobs:
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -146,9 +150,13 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -22,7 +22,7 @@ permissions:
|
||||
jobs:
|
||||
tests:
|
||||
name: S3 Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -25,7 +25,7 @@ defaults:
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -26,7 +26,7 @@ defaults:
|
||||
jobs:
|
||||
s3-versioning-tests:
|
||||
name: S3 Versioning Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
|
||||
s3-versioning-compatibility:
|
||||
name: S3 Versioning Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
|
||||
s3-cors-compatibility:
|
||||
name: S3 CORS Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
|
||||
s3-retention-tests:
|
||||
name: S3 Retention Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -266,7 +266,7 @@ jobs:
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -339,7 +339,7 @@ jobs:
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -398,7 +398,7 @@ jobs:
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -469,7 +469,7 @@ jobs:
|
||||
|
||||
s3-retention-worm:
|
||||
name: S3 Retention WORM Integration Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -519,7 +519,7 @@ jobs:
|
||||
|
||||
s3-versioning-stress:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
@@ -588,7 +588,7 @@ jobs:
|
||||
s3-tagging-tests:
|
||||
# CI job for S3 object tagging tests
|
||||
name: S3 Tagging Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -644,7 +644,7 @@ jobs:
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# Unit tests for IAM components
|
||||
iam-unit-tests:
|
||||
name: IAM Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
# S3 IAM integration tests with SeaweedFS services
|
||||
s3-iam-integration-tests:
|
||||
name: S3 IAM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
# Distributed IAM tests
|
||||
s3-iam-distributed-tests:
|
||||
name: S3 IAM Distributed Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -263,7 +263,7 @@ jobs:
|
||||
# Performance and stress tests
|
||||
s3-iam-performance-tests:
|
||||
name: S3 IAM Performance Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Dedicated job for Keycloak integration tests
|
||||
s3-keycloak-integration-tests:
|
||||
name: S3 Keycloak Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -21,7 +21,7 @@ permissions:
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
# Unit tests for policy components
|
||||
policy-unit-tests:
|
||||
name: S3 Policy Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
# S3 Policy Variables Integration Tests
|
||||
s3-policy-variables-tests:
|
||||
name: S3 Policy Variables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
# S3 Policy Enforcement Integration Tests
|
||||
s3-policy-enforcement-tests:
|
||||
name: S3 Policy Enforcement Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -317,7 +317,7 @@ jobs:
|
||||
# Trusted Proxy Detection Tests
|
||||
trusted-proxy-tests:
|
||||
name: Trusted Proxy Detection Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -405,7 +405,7 @@ jobs:
|
||||
# S3 Policy Shell Integration Tests
|
||||
s3-policy-shell-tests:
|
||||
name: S3 Policy Shell Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -24,7 +24,7 @@ permissions:
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -46,7 +46,7 @@ permissions:
|
||||
jobs:
|
||||
snowflake-s3compat-tests:
|
||||
name: Snowflake S3Compat API tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
WORK_DIR: /tmp/seaweedfs-snowflake-tests
|
||||
|
||||
@@ -20,7 +20,7 @@ permissions:
|
||||
jobs:
|
||||
s3-spark-issue-repro-tests:
|
||||
name: S3 Spark Issue Reproduction Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -36,7 +36,7 @@ defaults:
|
||||
jobs:
|
||||
s3-sse-integration-tests:
|
||||
name: S3 SSE Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
s3-sse-compatibility:
|
||||
name: S3 SSE Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
|
||||
s3-sse-metadata-persistence:
|
||||
name: S3 SSE Metadata Persistence Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -210,7 +210,7 @@ jobs:
|
||||
|
||||
s3-sse-copy-operations:
|
||||
name: S3 SSE Copy Operations Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -260,7 +260,7 @@ jobs:
|
||||
|
||||
s3-sse-multipart:
|
||||
name: S3 SSE Multipart Upload Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
|
||||
s3-sse-performance:
|
||||
name: S3 SSE Performance Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
# Only run performance tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
|
||||
@@ -361,7 +361,7 @@ jobs:
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -23,7 +23,7 @@ permissions:
|
||||
jobs:
|
||||
s3-tables-tests:
|
||||
name: S3 Tables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
|
||||
iceberg-catalog-tests:
|
||||
name: Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
|
||||
trino-iceberg-catalog-tests:
|
||||
name: Trino Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -227,7 +227,7 @@ jobs:
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -304,7 +304,7 @@ jobs:
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
|
||||
clickhouse-iceberg-catalog-tests:
|
||||
name: ClickHouse Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -470,7 +470,7 @@ jobs:
|
||||
|
||||
olake-iceberg-catalog-tests:
|
||||
name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -585,7 +585,7 @@ jobs:
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -657,7 +657,7 @@ jobs:
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -731,7 +731,7 @@ jobs:
|
||||
|
||||
risingwave-catalog-tests:
|
||||
name: RisingWave Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -806,7 +806,7 @@ jobs:
|
||||
|
||||
sts-integration-tests:
|
||||
name: STS Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -905,7 +905,7 @@ jobs:
|
||||
|
||||
lakekeeper-integration-tests:
|
||||
name: Lakekeeper Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -969,7 +969,7 @@ jobs:
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1046,7 +1046,7 @@ jobs:
|
||||
|
||||
lancedb-namespace-tests:
|
||||
name: LanceDB Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1120,7 +1120,7 @@ jobs:
|
||||
|
||||
table-lifecycle-tests:
|
||||
name: Table Lifecycle Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1189,7 +1189,7 @@ jobs:
|
||||
|
||||
duckdb-lance-tests:
|
||||
name: DuckDB Lance Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1268,7 +1268,7 @@ jobs:
|
||||
|
||||
spark-lance-namespace-tests:
|
||||
name: Spark Lance Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1346,7 +1346,7 @@ jobs:
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -1412,7 +1412,7 @@ jobs:
|
||||
|
||||
s3-tables-fmt-check:
|
||||
name: S3 Tables Format Check
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
@@ -1464,7 +1464,7 @@ jobs:
|
||||
|
||||
s3-tables-vet:
|
||||
name: S3 Tables Go Vet Check
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
basic-s3-tests:
|
||||
name: Basic S3 tests (KV store)
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -387,7 +387,7 @@ jobs:
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -558,7 +558,7 @@ jobs:
|
||||
|
||||
cors-tests:
|
||||
name: S3 CORS tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -684,7 +684,7 @@ jobs:
|
||||
|
||||
copy-tests:
|
||||
name: SeaweedFS Custom S3 Copy tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -787,7 +787,7 @@ jobs:
|
||||
|
||||
sql-store-tests:
|
||||
name: Basic S3 tests (SQL store)
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -29,7 +29,7 @@ permissions:
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -29,7 +29,7 @@ env:
|
||||
jobs:
|
||||
sftp-integration:
|
||||
name: SFTP Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -24,7 +24,7 @@ defaults:
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -9,7 +9,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
@@ -19,7 +18,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
@@ -37,7 +35,7 @@ env:
|
||||
jobs:
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -70,7 +68,7 @@ jobs:
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
@@ -143,77 +141,3 @@ jobs:
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
volume-server-rust-smoke:
|
||||
name: Volume Server Rust Smoke (${{ matrix.mode }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [exec, proxy, native]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Set up Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume launcher
|
||||
run: |
|
||||
cd rust/volume_server
|
||||
cargo build --release
|
||||
|
||||
- name: Run Rust-mode HTTP smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
|
||||
|
||||
- name: Run Rust-mode gRPC smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/volume-server-rust-smoke-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-rust-smoke-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-rust-smoke-logs-${{ matrix.mode }}
|
||||
path: /tmp/volume-server-rust-smoke-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Rust smoke summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.50"
|
||||
appVersion: "4.51"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.50.0
|
||||
version: 4.51.0
|
||||
@@ -166,6 +166,10 @@ message Entry {
|
||||
repeated FileChunk chunks = 3;
|
||||
FuseAttributes attributes = 4;
|
||||
map<string, bytes> extended = 5;
|
||||
// filer-local commit timestamp of the version stored here: the meta log
|
||||
// event's ts_ns for the mutation that produced this entry, or 0 for
|
||||
// entries written before the field existed
|
||||
int64 commit_ts_ns = 6;
|
||||
bytes hard_link_id = 7;
|
||||
int32 hard_link_counter = 8; // only exists in hard link meta data
|
||||
bytes content = 9; // if not empty, the file content
|
||||
@@ -281,6 +285,7 @@ message WriteCondition {
|
||||
IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value
|
||||
IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future
|
||||
IF_CHUNKS_EQUAL = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)
|
||||
IF_ENTRY_EQUAL = 10; // fail unless the stored entry equals expected_entry
|
||||
}
|
||||
// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current
|
||||
// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it
|
||||
@@ -309,6 +314,7 @@ message WriteCondition {
|
||||
string gate_key = 7; // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_value
|
||||
string gate_value = 8; // gate value (e.g. retention mode COMPLIANCE for governance bypass)
|
||||
repeated string fids = 9; // chunk fid strings for IF_CHUNKS_EQUAL
|
||||
Entry expected_entry = 10; // whole-entry comparison for IF_ENTRY_EQUAL
|
||||
}
|
||||
repeated Clause clauses = 1; // all must hold (logical AND)
|
||||
}
|
||||
@@ -323,6 +329,7 @@ enum FilerError {
|
||||
EXISTING_IS_FILE = 4; // cannot overwrite file with directory
|
||||
ENTRY_ALREADY_EXISTS = 5; // O_EXCL and entry already exists
|
||||
PRECONDITION_FAILED = 6; // WriteCondition not satisfied
|
||||
READ_ONLY = 7; // path matches a read-only storage rule (e.g. bucket over quota)
|
||||
}
|
||||
|
||||
// ObjectMutation is one entry-level change applied by ObjectTransaction. All
|
||||
@@ -351,6 +358,7 @@ message ObjectMutation {
|
||||
bytes content = 11; // PATCH_EXTENDED: new Entry.content when set_content
|
||||
bool touch_mtime = 12; // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
bool remove_empty_parent = 13; // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
bool keep_remote_object = 14; // DELETE: leave the object on the mounted remote storage
|
||||
}
|
||||
|
||||
// Recompute re-derives a pointer entry (directory/name on the mutation) from the
|
||||
@@ -388,6 +396,8 @@ message ObjectTransactionRequest {
|
||||
string condition_key = 6; // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)
|
||||
string route_key = 7; // ring key identifying the owner filer; a non-owner forwards the whole transaction to it
|
||||
bool is_moved = 8; // set on a forwarded transaction so the receiver applies it locally instead of forwarding again
|
||||
string request_id = 9; // idempotency token: the owner records the outcome and replays it on a retried request instead of re-applying
|
||||
string condition_alt_key = 10; // if the condition's entry is absent, evaluate it against this entry instead (e.g. null slot, else latest version)
|
||||
}
|
||||
|
||||
message ObjectTransactionResponse {
|
||||
@@ -477,6 +487,7 @@ message UpdateEntryRequest {
|
||||
// the write, under the filer's per-path lock. The caller must route the
|
||||
// key's writes to this entry's owner filer for the check to be authoritative.
|
||||
WriteCondition condition = 6;
|
||||
bool is_moved = 7; // set on a forwarded request so the owner applies it locally
|
||||
}
|
||||
message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
@@ -514,6 +525,7 @@ message DeleteEntryRequest {
|
||||
bool is_from_other_cluster = 7;
|
||||
repeated int32 signatures = 8;
|
||||
int64 if_not_modified_after = 9;
|
||||
bool keep_remote_object = 10;
|
||||
}
|
||||
|
||||
message DeleteEntryResponse {
|
||||
@@ -573,6 +585,9 @@ message AssignVolumeResponse {
|
||||
// fsync is the storage rule's fsync decision for the assigned path, so the
|
||||
// client can carry it onto the volume server upload request.
|
||||
bool fsync = 11;
|
||||
// error_code classifies error for callers that must branch on it; error
|
||||
// keeps the human-readable text for clients that predate the field.
|
||||
FilerError error_code = 12;
|
||||
}
|
||||
|
||||
message LookupVolumeRequest {
|
||||
@@ -709,6 +724,10 @@ message TraverseBfsMetadataRequest {
|
||||
message TraverseBfsMetadataResponse {
|
||||
string directory = 1;
|
||||
Entry entry = 2;
|
||||
// The sender's per-path commit timestamp (see meta_path_ts.go): the newest
|
||||
// event time this filer committed for the path, 0 when unrecorded so
|
||||
// subscribers fall back to the entry's mtime.
|
||||
int64 entry_ts_ns = 3;
|
||||
}
|
||||
|
||||
message LogEntry {
|
||||
|
||||
Generated
-7
@@ -1,7 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
@@ -1,6 +0,0 @@
|
||||
[package]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
@@ -1,245 +0,0 @@
|
||||
# Rust Volume Server Parity Implementation Plan
|
||||
|
||||
## Objective
|
||||
Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
|
||||
|
||||
## Current Focus (2026-02-16)
|
||||
- Program focus is now Rust implementation parity, not broad test expansion.
|
||||
- `test/volume_server` is treated as the parity gate.
|
||||
- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
|
||||
|
||||
## Current Status
|
||||
- Rust crate and launcher are in place.
|
||||
- Integration harness can run:
|
||||
- Go master + Go volume (default)
|
||||
- Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
|
||||
- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
|
||||
- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
|
||||
- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
|
||||
- `/status`, `/healthz`
|
||||
- `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*`
|
||||
- public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
|
||||
- admin unsupported verbs as `400` parity
|
||||
- Native Rust control-surface parity now also includes:
|
||||
- `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
|
||||
- absolute-form HTTP request-target normalization before native route matching
|
||||
- Native Rust HTTP data-path prevalidation now includes:
|
||||
- early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
|
||||
- slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
|
||||
- write-path error prevalidation for fid routes:
|
||||
- malformed multipart form-data requests without a boundary => `400`
|
||||
- requests containing `Content-MD5` header => `400` mismatch parity branch
|
||||
- request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
|
||||
- Native Rust API/storage handlers are not implemented yet.
|
||||
|
||||
## Parity Exit Criteria
|
||||
1. Native mode passes:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
|
||||
3. Rust mode defaults to native behavior for integration harness.
|
||||
4. Go-backend delegation is removed (or retained only as explicit fallback mode).
|
||||
|
||||
## Architecture Workstreams
|
||||
|
||||
### A. Runtime and Configuration Parity
|
||||
- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
|
||||
- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
|
||||
- [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
|
||||
- [ ] master target/config dir/read mode/throttling/JWT-related config
|
||||
- [ ] size/timeout controls and maintenance state defaults
|
||||
- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
|
||||
|
||||
### B. Native HTTP Surface
|
||||
- [ ] Admin/control endpoints:
|
||||
- [x] `GET /status` (native Rust in `native` mode)
|
||||
- [x] `GET /healthz` (native Rust in `native` mode)
|
||||
- [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
|
||||
- [x] static/UI endpoints currently exercised
|
||||
- [ ] Data read path parity:
|
||||
- [ ] fid parsing/path variants
|
||||
- [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
|
||||
- [ ] range handling (single/multi/invalid)
|
||||
- [ ] deleted reads, auth checks, read-mode branches
|
||||
- [ ] chunk-manifest and compression/image transformation branches
|
||||
- [ ] Data write/delete parity:
|
||||
- [ ] write success/unchanged/error paths
|
||||
- [ ] replication and file-size-limit paths
|
||||
- [ ] delete and chunk-manifest delete branches
|
||||
- [ ] Method/CORS/public-port parity for split admin/public behavior.
|
||||
|
||||
### C. Native gRPC Surface
|
||||
- [ ] Control-plane RPCs:
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
|
||||
- [ ] Data RPCs:
|
||||
- [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
|
||||
- [ ] `BatchDelete`, `ReadAllNeedles`
|
||||
- [ ] sync/copy/receive and status endpoints
|
||||
- [ ] Stream RPCs:
|
||||
- [ ] tail sender/receiver
|
||||
- [ ] vacuum streams
|
||||
- [ ] query streams
|
||||
- [ ] Advanced families:
|
||||
- [ ] erasure coding RPC set
|
||||
- [ ] tiering/remote fetch
|
||||
- [ ] scrub/query mode matrix
|
||||
|
||||
### D. Storage Compatibility Layer
|
||||
- [ ] Implement volume data/index handling compatible with Go on-disk format.
|
||||
- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
|
||||
- [ ] Match read/write/delete consistency and error mapping behavior.
|
||||
- [ ] Ensure EC metadata/data-path compatibility with existing files.
|
||||
|
||||
### E. Operational Hardening
|
||||
- [ ] Deterministic startup/readiness and shutdown semantics.
|
||||
- [ ] Log/error parity sufficient for debugging and CI triage.
|
||||
- [ ] Concurrency/timeout behavior alignment for throttling and streams.
|
||||
- [ ] Performance baseline checks vs Go for key flows.
|
||||
|
||||
## Milestone Plan
|
||||
|
||||
### M0 (Completed): Harness + Launcher Transition
|
||||
- [x] Rust launcher integrated into harness.
|
||||
- [x] Proxy mode full-suite validation with Go backend delegation.
|
||||
|
||||
### M1: Native Skeleton (Control Plane First)
|
||||
- [ ] `native` mode boots and serves:
|
||||
- [x] `/status`, `/healthz`
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- Gate:
|
||||
- [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
|
||||
|
||||
### M2: Native Core Data Paths
|
||||
- [ ] Native HTTP read/write/delete baseline parity.
|
||||
- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
|
||||
- Gate:
|
||||
- core HTTP and gRPC data suites pass in `native` mode.
|
||||
|
||||
### M3: Native Stream + Copy/Sync
|
||||
- [ ] Tail/copy/receive/sync paths in native mode.
|
||||
- Gate:
|
||||
- stream/copy families pass in `native` mode.
|
||||
|
||||
### M4: Native Advanced Feature Families
|
||||
- [ ] EC, tiering, scrub/query advanced branches.
|
||||
- Gate:
|
||||
- full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
|
||||
|
||||
### M5: CI/Cutover
|
||||
- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
|
||||
- [x] Make native mode default for Rust integration runs.
|
||||
- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
|
||||
|
||||
## Immediate Next Steps
|
||||
1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
|
||||
2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
|
||||
3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
|
||||
4. Add mismatch triage notes for each API moved from delegation to native implementation.
|
||||
|
||||
## Risk Register
|
||||
- On-disk format mismatch risk:
|
||||
- Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
|
||||
- Behavioral drift in edge branches:
|
||||
- Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
|
||||
- Stream/concurrency semantic mismatch:
|
||||
- Mitigation: stabilize with focused interruption/timeout parity tests.
|
||||
|
||||
## Progress Log
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust launcher integration (`exec`) and harness wiring.
|
||||
- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
|
||||
- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy supervisor mode and validated full integration suite.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `a7f50d23b`, `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Re-focused plan from test expansion to native Rust implementation parity.
|
||||
- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
|
||||
- Commits: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
|
||||
- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating parity
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
|
||||
- public non-read methods as `200` no-op passthrough parity
|
||||
- admin unsupported methods as `400` parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Hardened native Rust HTTP control-path parity:
|
||||
- `/healthz` now mirrors backend health status (`200`/`503`) in native mode
|
||||
- absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native malformed fid-route validation in `native` mode:
|
||||
- GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
|
||||
- valid fid-route requests still delegate to backend handlers for storage/data execution paths
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
|
||||
- multipart form uploads without boundary now return native `400`
|
||||
- `Content-MD5` requests now return native `400` mismatch branch
|
||||
- `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened native fid-route parser to cover slash-form URLs:
|
||||
- `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
|
||||
- added explicit non-fid exclusions for control/stats/static paths to preserve route parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `e66983e9b`
|
||||
File diff suppressed because it is too large.
Load diff
Generated
+1
@@ -4601,6 +4601,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex",
|
||||
"http-body 1.0.1",
|
||||
"httpdate",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"image",
|
||||
|
||||
@@ -51,6 +51,7 @@ prost-types = "0.14"
|
||||
# HTTP server
|
||||
axum = { version = "0.8", features = ["multipart"] }
|
||||
http-body = "1"
|
||||
httpdate = "1"
|
||||
hyper = { version = "1", features = ["full"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "service", "server-auto", "http1", "http2"] }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
|
||||
@@ -1120,6 +1120,10 @@ async fn get_or_head_handler_inner(
|
||||
};
|
||||
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
// A Range whose If-Range no longer matches is ignored and the whole
|
||||
// content is sent, so a resumed download restarts instead of mixing
|
||||
// two versions.
|
||||
let range = range.filter(|_| if_range_matches(&headers, &etag, &last_modified_str));
|
||||
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
|
||||
// A request that ends here was answered by the meta read alone.
|
||||
if let Some(io_errors) = strategy.io_errors() {
|
||||
@@ -1167,14 +1171,20 @@ async fn get_or_head_handler_inner(
|
||||
return head_from_meta_response(&info, response_headers);
|
||||
}
|
||||
ReadStrategy::RangeFromSource(info) => {
|
||||
return range_from_source_response(
|
||||
&state,
|
||||
range.as_deref().unwrap_or_default(),
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await;
|
||||
return match range.as_deref() {
|
||||
Some(range) => {
|
||||
range_from_source_response(
|
||||
&state,
|
||||
range,
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// If-Range dropped the range; serve the whole content.
|
||||
None => stream_response(&state, info, response_headers, track_download),
|
||||
};
|
||||
}
|
||||
ReadStrategy::Buffered => {}
|
||||
}
|
||||
@@ -1557,6 +1567,31 @@ fn not_modified_response(
|
||||
None
|
||||
}
|
||||
|
||||
/// If-Range (RFC 9110 13.1.5): an entity-tag by strong comparison, or a date
|
||||
/// equal to Last-Modified. The response etag here is always a strong tag.
|
||||
fn if_range_matches(headers: &HeaderMap, etag: &str, last_modified_str: &Option<String>) -> bool {
|
||||
let Some(raw) = headers.get(header::IF_RANGE) else {
|
||||
return true;
|
||||
};
|
||||
let Ok(if_range) = raw.to_str() else {
|
||||
return false;
|
||||
};
|
||||
let if_range = if_range.trim();
|
||||
if if_range.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if if_range.starts_with('"') || if_range.starts_with("W/") {
|
||||
return if_range == etag;
|
||||
}
|
||||
let Ok(date) = httpdate::parse_http_date(if_range) else {
|
||||
return false;
|
||||
};
|
||||
last_modified_str
|
||||
.as_deref()
|
||||
.and_then(|lm| httpdate::parse_http_date(lm).ok())
|
||||
.is_some_and(|lm| date == lm)
|
||||
}
|
||||
|
||||
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
|
||||
fn etag_list_matches(header: &str, etag: &str) -> bool {
|
||||
if header.trim() == "*" {
|
||||
@@ -5767,6 +5802,53 @@ mod tests {
|
||||
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_if_range_matches() {
|
||||
let etag = "\"v2\"".to_string();
|
||||
let last_modified_str = Some("Sat, 24 Oct 2026 22:01:04 GMT".to_string());
|
||||
let cases: Vec<(Option<&str>, bool)> = vec![
|
||||
(None, true),
|
||||
(Some("\"v2\""), true),
|
||||
(Some("Sat, 24 Oct 2026 22:01:04 GMT"), true),
|
||||
(Some("Saturday, 24-Oct-26 22:01:04 GMT"), true),
|
||||
(Some("Sat Oct 24 22:01:04 2026"), true),
|
||||
(Some("\"v1\""), false),
|
||||
(Some("W/\"v2\""), false),
|
||||
(Some("Sat, 24 Oct 2026 22:01:03 GMT"), false),
|
||||
(Some("not a date"), false),
|
||||
];
|
||||
for (if_range, want) in cases {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(v) = if_range {
|
||||
headers.insert(header::IF_RANGE, v.parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
if_range_matches(&headers, &etag, &last_modified_str),
|
||||
want,
|
||||
"{if_range:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// A present value that is not valid UTF-8 fails the condition.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
header::HeaderValue::from_bytes(b"\"\xff\xfev2\"").unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &last_modified_str));
|
||||
|
||||
// No Last-Modified on the representation: a date validator fails,
|
||||
// an entity-tag still applies.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::IF_RANGE, "\"v2\"".parse().unwrap());
|
||||
assert!(if_range_matches(&headers, &etag, &None));
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
"Sat, 24 Oct 2026 22:01:04 GMT".parse().unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &None));
|
||||
}
|
||||
|
||||
/// A GET whose needle read is parked must not hold the store lock: a
|
||||
/// writer, here a real append to the same volume, must get through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
@@ -34,6 +34,7 @@ struct ListTablesResponse {
|
||||
pub struct NamespaceClient {
|
||||
base_url: String,
|
||||
http: reqwest::Client,
|
||||
api_key: Option<String>,
|
||||
}
|
||||
|
||||
/// A namespace call that has not answered by now is not going to. Without this
|
||||
@@ -41,12 +42,23 @@ pub struct NamespaceClient {
|
||||
/// slot open forever, and the sweep never finishes.
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const MAX_REDIRECT_HOPS: usize = 10;
|
||||
|
||||
impl NamespaceClient {
|
||||
pub fn new(base_url: impl Into<String>) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
// reqwest forwards custom headers on a follow, so only a same-origin
|
||||
// hop is allowed: anywhere else would carry x-api-key to a listener
|
||||
// the credential was never meant for.
|
||||
.redirect(reqwest::redirect::Policy::custom(|attempt| {
|
||||
if may_follow(attempt.previous(), attempt.url()) {
|
||||
attempt.follow()
|
||||
} else {
|
||||
attempt.stop()
|
||||
}
|
||||
}))
|
||||
.build()
|
||||
// The builder only fails on a bad TLS backend, which would break
|
||||
// every call anyway; a client with no timeouts is worse than a panic
|
||||
@@ -58,6 +70,32 @@ impl NamespaceClient {
|
||||
Self {
|
||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||
http,
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The catalog accepts an S3 credential as `x-api-key`; where the worker
|
||||
/// already holds one for storage it authenticates the namespace calls too.
|
||||
pub fn with_fallback(mut self, fallback: &crate::dataset::FallbackOptions) -> Self {
|
||||
if let (Some(access), Some(secret)) = (
|
||||
fallback.get("aws_access_key_id"),
|
||||
fallback.get("aws_secret_access_key"),
|
||||
) {
|
||||
if !protected_transport(&self.base_url) {
|
||||
tracing::warn!(
|
||||
"catalog url {} exposes the x-api-key credential in cleartext",
|
||||
self.base_url
|
||||
);
|
||||
}
|
||||
self.api_key = Some(format!("{access}:{secret}"));
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match &self.api_key {
|
||||
Some(key) => request.header("x-api-key", key),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +103,7 @@ impl NamespaceClient {
|
||||
pub async fn list_all_tables(&self) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/table", self.base_url);
|
||||
let response: ListTablesResponse = self
|
||||
.http
|
||||
.get(&url)
|
||||
.authed(self.http.get(&url))
|
||||
.send()
|
||||
.await
|
||||
.context("list tables")?
|
||||
@@ -80,8 +117,7 @@ impl NamespaceClient {
|
||||
pub async fn list_namespaces(&self, id: &[String]) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/namespace/{}/list", self.base_url, encode_id(id));
|
||||
let response: ListNamespacesResponse = self
|
||||
.http
|
||||
.get(&url)
|
||||
.authed(self.http.get(&url))
|
||||
.send()
|
||||
.await
|
||||
.context("list namespaces")?
|
||||
@@ -98,8 +134,7 @@ impl NamespaceClient {
|
||||
let url = format!("{}/v1/table/{}/describe", self.base_url, encode_id(id));
|
||||
let body = serde_json::json!({ "id": id, "vend_credentials": true });
|
||||
let description: TableDescription = self
|
||||
.http
|
||||
.post(&url)
|
||||
.authed(self.http.post(&url))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
@@ -111,6 +146,41 @@ impl NamespaceClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a hop bound a catalog that redirects to itself keeps the request
|
||||
/// alive until the call timeout, so the custom policy keeps the limit the
|
||||
/// default one had.
|
||||
fn may_follow(previous: &[reqwest::Url], next: &reqwest::Url) -> bool {
|
||||
previous.len() < MAX_REDIRECT_HOPS
|
||||
&& previous.last().is_some_and(|prev| same_origin(prev, next))
|
||||
}
|
||||
|
||||
/// A redirect stays within an origin when scheme, host, and port all match, so
|
||||
/// a catalog cannot bounce the api key onto a downgraded or foreign listener.
|
||||
fn same_origin(from: &reqwest::Url, to: &reqwest::Url) -> bool {
|
||||
from.scheme() == to.scheme()
|
||||
&& from.host_str() == to.host_str()
|
||||
&& from.port_or_known_default() == to.port_or_known_default()
|
||||
}
|
||||
|
||||
/// A catalog url protects the api key when the hop is TLS or never leaves the
|
||||
/// host; anything else sends it where the network can read it.
|
||||
fn protected_transport(base_url: &str) -> bool {
|
||||
let Ok(url) = reqwest::Url::parse(base_url) else {
|
||||
return false;
|
||||
};
|
||||
if url.scheme() == "https" {
|
||||
return true;
|
||||
}
|
||||
match url.host_str() {
|
||||
Some("localhost") => true,
|
||||
Some(host) => host
|
||||
.trim_matches(['[', ']'])
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|ip| ip.is_loopback()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_id(id: &[String]) -> String {
|
||||
if id.is_empty() {
|
||||
DELIMITER.to_string()
|
||||
@@ -127,3 +197,56 @@ pub fn parse_id(encoded: &str) -> Vec<String> {
|
||||
.map(|part| part.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin};
|
||||
|
||||
#[test]
|
||||
fn may_follow_bounds_hops_within_the_origin() {
|
||||
let url = |u: &str| reqwest::Url::parse(u).unwrap();
|
||||
let first = [url("https://c.local/ns")];
|
||||
let next = url("https://c.local/other");
|
||||
assert!(may_follow(&first, &next));
|
||||
assert!(!may_follow(&[], &next));
|
||||
assert!(!may_follow(&first, &url("http://c.local/other")));
|
||||
let full: Vec<_> = (0..MAX_REDIRECT_HOPS)
|
||||
.map(|i| url(&format!("https://c.local/{i}")))
|
||||
.collect();
|
||||
assert!(!may_follow(&full, &next));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_origin_rejects_scheme_and_port_hops() {
|
||||
for (from, to, expected) in [
|
||||
("https://c.local/ns", "https://c.local/other", true),
|
||||
("http://c.local", "http://c.local:80/next", true),
|
||||
("https://c.local", "http://c.local/next", false),
|
||||
("https://c.local:8443", "https://c.local/next", false),
|
||||
("https://c.local", "https://other.local/next", false),
|
||||
] {
|
||||
let from = reqwest::Url::parse(from).unwrap();
|
||||
let to = reqwest::Url::parse(to).unwrap();
|
||||
assert_eq!(same_origin(&from, &to), expected, "{from} -> {to}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_transport_covers_tls_and_loopback() {
|
||||
for url in [
|
||||
"https://catalog.internal:8443",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://localhost",
|
||||
"http://[::1]:8080",
|
||||
] {
|
||||
assert!(protected_transport(url), "{url} should be protected");
|
||||
}
|
||||
for url in [
|
||||
"http://catalog.internal",
|
||||
"http://127.0.0.1.evil.test",
|
||||
"not a url",
|
||||
] {
|
||||
assert!(!protected_transport(url), "{url} should warn");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ impl CleanupVersionsHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ impl CompactHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ impl OptimizeIndicesHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ impl SortHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ impl PreviewProvider for LancePreview {
|
||||
}
|
||||
|
||||
async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
|
||||
let client = NamespaceClient::new(self.namespace_url.clone());
|
||||
let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback);
|
||||
let table = dataset::open(&client, object_id, &self.fallback).await?;
|
||||
|
||||
let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
|
||||
|
||||
@@ -244,7 +244,13 @@ func (env *environment) tableBucketExists(t *testing.T, bucket string) bool {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s:%d/v1/namespace/%s/exists", env.bindIP, env.lancePort, bucket)
|
||||
resp, err := http.Post(url, "application/json", strings.NewReader("{}"))
|
||||
req, err := http.NewRequest(http.MethodPost, url, strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("build the namespace exists request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", env.accessKey+":"+env.secretKey)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("ask the namespace whether %s exists: %v", bucket, err)
|
||||
}
|
||||
|
||||
@@ -53,7 +53,13 @@ def main():
|
||||
"aws_region": "us-east-1",
|
||||
}
|
||||
|
||||
ns = ln.connect("rest", {"uri": args.namespace_url})
|
||||
ns = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
ns.create_namespace(ln.CreateNamespaceRequest(id=[args.bucket], mode="EXIST_OK"))
|
||||
ns.create_namespace(
|
||||
ln.CreateNamespaceRequest(id=[args.bucket, args.namespace], mode="EXIST_OK")
|
||||
|
||||
@@ -45,7 +45,15 @@ def seed_table(namespace_url, storage, bucket, namespace, table, rows):
|
||||
LanceDB reads through the catalog; the writing half is pylance, because the
|
||||
namespace records where a table lives and does not carry its data.
|
||||
"""
|
||||
ns = ln.connect("rest", {"uri": namespace_url})
|
||||
ns = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": namespace_url,
|
||||
"header.x-api-key": (
|
||||
f"{storage['aws_access_key_id']}:{storage['aws_secret_access_key']}"
|
||||
),
|
||||
},
|
||||
)
|
||||
ns.create_namespace(
|
||||
ln.CreateNamespaceRequest(id=[bucket], mode="EXIST_OK")
|
||||
)
|
||||
@@ -98,7 +106,12 @@ def main():
|
||||
|
||||
print(f"lancedb {lancedb.__version__} connecting to {args.namespace_url}")
|
||||
db = lancedb.connect_namespace(
|
||||
"rest", {"uri": args.namespace_url}, storage_options=storage
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
storage_options=storage,
|
||||
)
|
||||
|
||||
# 1. The catalog is browsable: the bucket is a namespace, and the table is
|
||||
@@ -190,7 +203,10 @@ def main():
|
||||
# declare-and-write - rather than a hang, a 404, or a half-made table.
|
||||
pushdown = lancedb.connect_namespace(
|
||||
"rest",
|
||||
{"uri": args.namespace_url},
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
storage_options=storage,
|
||||
namespace_client_pushdown_operations=["CreateTable"],
|
||||
)
|
||||
|
||||
@@ -41,6 +41,7 @@ deployment uses whatever address the gateway answers on:
|
||||
spark.sql.catalog.lance org.lance.spark.LanceNamespaceSparkCatalog
|
||||
spark.sql.catalog.lance.impl rest
|
||||
spark.sql.catalog.lance.uri http://seaweed:9101
|
||||
spark.sql.catalog.lance.headers.x-api-key <access-key>:<secret-key>
|
||||
spark.sql.catalog.lance.storage.aws_endpoint http://seaweed:8333
|
||||
spark.sql.catalog.lance.storage.allow_http true
|
||||
spark.sql.catalog.lance.storage.aws_access_key_id …
|
||||
|
||||
@@ -46,6 +46,10 @@ def main():
|
||||
.config("spark.sql.catalog.lance", "org.lance.spark.LanceNamespaceSparkCatalog")
|
||||
.config("spark.sql.catalog.lance.impl", "rest")
|
||||
.config("spark.sql.catalog.lance.uri", args.namespace_url)
|
||||
.config(
|
||||
"spark.sql.catalog.lance.headers.x-api-key",
|
||||
f"{args.access_key}:{args.secret_key}",
|
||||
)
|
||||
.config("spark.sql.catalog.lance.storage.aws_endpoint", args.s3_endpoint)
|
||||
.config("spark.sql.catalog.lance.storage.allow_http", "true")
|
||||
.config("spark.sql.catalog.lance.storage.aws_access_key_id", args.access_key)
|
||||
|
||||
@@ -239,7 +239,13 @@ func (env *environment) tableBucketExists(t *testing.T, bucket string) bool {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s:%d/v1/namespace/%s/exists", env.bindIP, env.lancePort, bucket)
|
||||
resp, err := http.Post(url, "application/json", strings.NewReader("{}"))
|
||||
req, err := http.NewRequest(http.MethodPost, url, strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("build the namespace exists request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", env.accessKey+":"+env.secretKey)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("ask the namespace whether %s exists: %v", bucket, err)
|
||||
}
|
||||
|
||||
@@ -99,7 +99,8 @@ def connect(args):
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": f"s3://{args.bucket}/",
|
||||
"prefix": args.bucket,
|
||||
"auth": {"type": "noop"},
|
||||
"oauth2-server-uri": f"{args.catalog_url}/v1/oauth/tokens",
|
||||
"credential": f"{args.access_key}:{args.secret_key}",
|
||||
"s3.endpoint": args.s3_endpoint,
|
||||
"s3.access-key-id": args.access_key,
|
||||
"s3.secret-access-key": args.secret_key,
|
||||
|
||||
@@ -83,7 +83,13 @@ def tally(dataset):
|
||||
|
||||
def resolve(args):
|
||||
"""Ask the namespace where the table lives, the way the worker does."""
|
||||
namespace = ln.connect("rest", {"uri": args.namespace_url})
|
||||
namespace = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
table_id = [args.bucket, args.namespace, args.table]
|
||||
described = namespace.describe_table(ln.DescribeTableRequest(id=table_id))
|
||||
return namespace, table_id, described.location, storage_options(args, described)
|
||||
@@ -115,7 +121,13 @@ def main():
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.phase == "write":
|
||||
namespace = ln.connect("rest", {"uri": args.namespace_url})
|
||||
namespace = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
for parent in ([args.bucket], [args.bucket, args.namespace]):
|
||||
namespace.create_namespace(ln.CreateNamespaceRequest(id=parent, mode="EXIST_OK"))
|
||||
table_id = [args.bucket, args.namespace, args.table]
|
||||
|
||||
@@ -3,12 +3,6 @@
|
||||
## Goal
|
||||
Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior.
|
||||
|
||||
## Current Program Focus (2026-02-16)
|
||||
- Primary execution focus has shifted to implementing native Rust volume-server parity.
|
||||
- This integration suite is now the parity gate for Rust implementation work.
|
||||
- New tests should be added only when native Rust implementation reveals uncovered Go behavior that is not yet captured.
|
||||
- Rust implementation roadmap lives in `rust/volume_server/DEV_PLAN.md`.
|
||||
|
||||
## Hard Requirements
|
||||
- Tests live under `test/volume_server`.
|
||||
- Tests are written in Go.
|
||||
@@ -1133,202 +1127,3 @@ Update this section during implementation:
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending.
|
||||
- Commit: `d1e5f390a`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust volume-server migration bootstrap wiring.
|
||||
- APIs covered: Harness now supports Go master + selectable volume binary (`VOLUME_SERVER_IMPL=rust` or `VOLUME_SERVER_BINARY`), with Rust-mode smoke coverage for representative HTTP/gRPC integration tests.
|
||||
- Profiles covered: P1 smoke and default Go matrix unchanged.
|
||||
- Gaps introduced/remaining: native Rust handler implementations are tracked in `rust/volume_server/DEV_PLAN.md`; current Rust mode is a compatibility launcher phase.
|
||||
- Commit: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Validated Rust-mode compatibility path with full integration suites.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust`.
|
||||
- Profiles covered: existing integration matrix in Rust launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint and storage/RPC implementation remains pending; current mode delegates execution to Go volume server.
|
||||
- Commit: `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy-supervision launcher mode and validated all volume-server integration suites in proxy mode.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust` and `VOLUME_SERVER_RUST_MODE=proxy`.
|
||||
- Profiles covered: existing integration matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC logic remains pending; current proxy mode still delegates backend handlers to Go.
|
||||
- Commit: `a7f50d23b`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded GitHub CI Rust smoke job to run both launcher modes (`exec` and `proxy`).
|
||||
- APIs covered: representative HTTP/gRPC smoke flows now validated in CI for both Rust launcher execution paths.
|
||||
- Profiles covered: CI smoke profile for P1 representative calls.
|
||||
- Gaps introduced/remaining: CI still runs Rust mode as smoke scope; full-suite Rust-mode CI remains optional due runtime cost.
|
||||
- Commit: `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added JWT UI override integration coverage with explicit `access.ui=true` profile support.
|
||||
- APIs covered: `/ui/index.html` behavior under JWT profile now verifies both default auth-gated path (`401`) and explicit UI-enable override path (`200` with rendered UI).
|
||||
- Profiles covered: P3 and P3+`access.ui=true`.
|
||||
- Gaps introduced/remaining: UI exposure behavior under JWT profiles is now covered for both secured and override branches.
|
||||
- Commit: `de974c05d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added oversized upload integration coverage using explicit `-fileSizeLimitMB` profile configuration.
|
||||
- APIs covered: HTTP write path now verifies payloads larger than configured file-size limit are rejected (`400`) with limit-related error context.
|
||||
- Profiles covered: P1-derived profile with `FileSizeLimitMB=1`.
|
||||
- Gaps introduced/remaining: file-size limit rejection branch is now covered; replicate-write failure and cancellation (`499`) branches remain pending.
|
||||
- Commit: `4d61cbdee`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added gRPC EC-only metadata-read unsupported-path coverage.
|
||||
- APIs covered: `ReadNeedleMeta` now verifies the explicit unsupported branch when a volume is unmounted and only EC shards are mounted.
|
||||
- Profiles covered: P1 with EC shard generation/mount lifecycle setup.
|
||||
- Gaps introduced/remaining: low-level read metadata fault-injection and transport interruption branches remain pending.
|
||||
- Commit: `37bf9b5eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added deterministic replicated-write failure coverage for unmet replication requirements.
|
||||
- APIs covered: HTTP write path now verifies replication error handling (`500`) and no-local-commit outcome (`404` on follow-up read) when volume replication is configured (`001`) but replica set cannot be satisfied.
|
||||
- Profiles covered: P1-derived single-node profile with per-volume replication override.
|
||||
- Gaps introduced/remaining: replicated-write failure branch is now covered; cancellation (`499`) and deeper transport fault-injection branches remain pending.
|
||||
- Commit: `4835d3443`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added EC-backed `BatchDelete` positive-path integration coverage.
|
||||
- APIs covered: `BatchDelete` now validates successful EC-needle deletion (`202`) with post-delete `VolumeEcShardRead` deleted-marker verification (`IsDeleted=true`).
|
||||
- Profiles covered: P1 with EC shard generate/mount lifecycle; includes test-local gRPC offset proxy for EC internal dial-path parity.
|
||||
- Gaps introduced/remaining: EC `BatchDelete` success branch is now covered; deeper distributed failure-injection permutations remain pending.
|
||||
- Commit: `1bb40b6bc`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded HTTP conditional-header matrix for `HEAD` requests with combined header scenarios.
|
||||
- APIs covered: `HEAD` now verifies `If-Modified-Since` precedence over mismatched `If-None-Match`, and `If-None-Match` precedence when IMS is stale but ETag matches.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: combined conditional-header precedence paths for `HEAD`/`GET` are now explicitly covered.
|
||||
- Commit: `ed23e290f`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added multi-volume success-stream coverage for `ReadAllNeedles`.
|
||||
- APIs covered: `ReadAllNeedles` now verifies happy-path streaming across two existing volumes in one request, with per-volume payload validation.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: multi-volume stream boundary coverage is now present for successful reads; transport interruption branches remain pending.
|
||||
- Commit: `ab95a6ef1`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded split-port unsupported HTTP method matrix with `MKCOL` coverage.
|
||||
- APIs covered: admin/public parity for `MKCOL` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification.
|
||||
- Profiles covered: P2.
|
||||
- Gaps introduced/remaining: unsupported-method sampling now covers `PATCH`, `TRACE`, `PROPFIND`, `CONNECT`, and `MKCOL`; cancellation/transport fault paths remain the main unaddressed area.
|
||||
- Commit: `2ab30900d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Hardened framework port allocation to keep volume ports within `10000..55535`.
|
||||
- APIs covered: test harness now guarantees valid `admin+10000` gRPC offset ranges, eliminating EC batch-delete flake scenarios that rely on offset dialing.
|
||||
- Profiles covered: all profiles using shared framework port allocation.
|
||||
- Gaps introduced/remaining: no new API gaps; transport cancellation/fault-injection branches remain the primary uncovered area.
|
||||
- Commit: `90e82b15c`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Re-validated full Rust proxy-mode integration suite after latest HTTP/gRPC coverage additions and framework hardening.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages.
|
||||
- Profiles covered: existing matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC implementation remains pending (current Rust mode still delegates to Go backend handlers).
|
||||
- Commit: `6bb9d8bac`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added tail sender stream-cancellation interruption coverage.
|
||||
- APIs covered: `VolumeTailSender` now verifies client-side context cancellation behavior (`codes.Canceled`) after stream start/heartbeat.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: tail transport interruption coverage is now partially closed; receiver-side interruption and deeper network fault injection remain pending.
|
||||
- Commit: `27a80f760`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added receiver-side tail stream interruption coverage for unavailable source node.
|
||||
- APIs covered: `VolumeTailReceiver` now verifies source connect/dial failure branch when `SourceVolumeServer` is unreachable.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: both sender and receiver transport interruption branches are now covered at integration level; deeper injected network faults remain pending.
|
||||
- Commit: `a5864c3eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded query CSV matrix with explicit CSV-payload parity coverage.
|
||||
- APIs covered: `Query` now verifies current CSV-input behavior (no streamed rows / immediate EOF) even when source needle content is valid CSV text.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: CSV parsing/selection implementation still absent in current server behavior; integration parity now locks the current no-output semantics for both JSON and CSV payload shapes.
|
||||
- Commit: `a12dd5f8d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded ping unreachable-target matrix with volume-server target coverage.
|
||||
- APIs covered: `Ping` now explicitly verifies unreachable-target error wrapping for `target_type=volumeServer`, alongside existing master/filer variants.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: ping target-type matrix is covered for success and unreachable branches across master/filer/volume-server.
|
||||
- Commit: `b9fbb85af`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded deleted-read HTTP matrix with `HEAD` parity coverage.
|
||||
- APIs covered: `HEAD` with `readDeleted=true` now validates current behavior (`200`, empty body, payload-size `Content-Length`) after delete.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: deleted-read parity now covers both `GET` and `HEAD` semantics on local-volume path.
|
||||
- Commit: `cc80ad364`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Shifted planning priority to native Rust implementation parity.
|
||||
- APIs covered: no new API additions in this plan entry; integration suite remains the validation gate.
|
||||
- Profiles covered: unchanged.
|
||||
- Gaps introduced/remaining: primary remaining gap is native Rust handler/storage/RPC implementation replacing Go backend delegation.
|
||||
- Commit: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust launcher `native` mode bootstrap, made it the default Rust launcher mode, and expanded CI Rust smoke matrix coverage to include `native`.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated in `VOLUME_SERVER_RUST_MODE=native`; default Rust launcher path (`VOLUME_SERVER_IMPL=rust`) smoke-validated for HTTP and gRPC control tests.
|
||||
- Profiles covered: existing HTTP/gRPC matrix in native launcher mode; default-mode smoke checks on P1 control flows.
|
||||
- Gaps introduced/remaining: native Rust API/storage/RPC handlers still pending; current `native` mode remains a delegation bootstrap while parity replacement proceeds.
|
||||
- Commit: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP control handlers in `native` mode for `/status` and `/healthz` (admin listener path), with proxy fallback retained for remaining endpoints.
|
||||
- APIs covered: `/status` and `/healthz` now served by Rust-native code path in `VOLUME_SERVER_RUST_MODE=native`; full HTTP and gRPC integration suites re-validated.
|
||||
- Profiles covered: full existing HTTP/gRPC matrix under native mode.
|
||||
- Gaps introduced/remaining: gRPC control/data APIs and most HTTP data/static/auth paths remain delegated and require incremental native replacement.
|
||||
- Commit: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust HTTP `OPTIONS` handling for admin/public listeners in `VOLUME_SERVER_RUST_MODE=native`.
|
||||
- APIs covered: admin/public `OPTIONS` method allow-list and CORS origin handling now served from Rust-native path; full HTTP and gRPC packages re-validated.
|
||||
- Profiles covered: P2 split-port CORS/OPTIONS profiles plus full existing matrix in native mode.
|
||||
- Gaps introduced/remaining: static/UI/auth and data-path HTTP handlers, plus gRPC control/data/stream handlers, remain delegated and pending native replacement.
|
||||
- Commit: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented broader native Rust HTTP surface in `native` mode:
|
||||
- `/ui/index.html` auth-gating parity (JWT + access.ui handling)
|
||||
- static asset paths (`/favicon.ico`, `/seaweedfsstatic/*`)
|
||||
- public non-read method no-op parity (`200`) and admin unsupported-method rejection parity (`400`)
|
||||
- APIs covered: UI/static/admin/public control-surface behavior now served by Rust-native path; full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated.
|
||||
- Profiles covered: P1/P2/P3 variants touched by UI/static/CORS/method behavior, plus full matrix in native mode.
|
||||
- Gaps introduced/remaining: HTTP data-path handlers and all gRPC handlers remain delegated and still require native replacement.
|
||||
- Commit: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Improved native Rust control-path parity for health/status and request-target parsing.
|
||||
- APIs covered: native `/healthz` now mirrors backend service health transitions (`200`/`503`) and native route matching now handles absolute-form HTTP request targets before path dispatch.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode (`VOLUME_SERVER_IMPL=rust`, `VOLUME_SERVER_RUST_MODE=native`).
|
||||
- Gaps introduced/remaining: core HTTP data handlers and all gRPC RPC handlers are still delegated and remain the primary native implementation gap.
|
||||
- Commit: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native malformed fid-route validation ahead of delegated data handlers.
|
||||
- APIs covered: GET/HEAD/POST/PUT fid-shaped paths now return native `400` for invalid vid/fid tokens (including invalid-read and invalid-write path variants) while valid routes continue through delegated storage handlers.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode, plus targeted invalid-path parity runs.
|
||||
- Gaps introduced/remaining: data-path success branches and all gRPC handler bodies remain delegated; native replacement work continues on those core execution paths.
|
||||
- Commit: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native write error prevalidation for delegated fid routes.
|
||||
- APIs covered: native path now rejects malformed multipart boundary requests, `Content-MD5` writes, and over-limit `Content-Length` writes (using `-fileSizeLimitMB`) with `400` parity behavior before delegation.
|
||||
- Profiles covered: P1 default and P1 with explicit file-size limit profile, plus full existing HTTP/gRPC native-mode matrix revalidation.
|
||||
- Gaps introduced/remaining: positive write/read/delete data handlers and all gRPC handler bodies are still delegated; native implementation continues incrementally.
|
||||
- Commit: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened Rust-native malformed-fid handling to slash-form read/write URL shapes with reserved-route exclusions.
|
||||
- APIs covered: `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` malformed cases now hit native `400` validation path, while `/status`, `/healthz`, `/ui/index.html`, `/stats/*`, and static asset routes remain excluded from fid parsing.
|
||||
- Profiles covered: full existing HTTP/gRPC native-mode matrix, plus targeted admin/read-path malformed-slash validation run.
|
||||
- Gaps introduced/remaining: successful data-path read/write/delete handlers and all gRPC method bodies remain delegated and are still the primary native implementation gap.
|
||||
- Commit: `e66983e9b`
|
||||
@@ -1,10 +1,7 @@
|
||||
.PHONY: test-volume-server test-volume-server-short test-volume-server-rust-smoke
|
||||
.PHONY: test-volume-server test-volume-server-short
|
||||
|
||||
test-volume-server:
|
||||
go test ./test/volume_server/... -v
|
||||
|
||||
test-volume-server-short:
|
||||
go test ./test/volume_server/... -short -v
|
||||
|
||||
test-volume-server-rust-smoke:
|
||||
VOLUME_SERVER_IMPL=rust go test ./test/volume_server/http ./test/volume_server/grpc -run 'TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip|TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus' -v
|
||||
@@ -15,9 +15,6 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
## Optional environment variables
|
||||
|
||||
- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
|
||||
- `VOLUME_SERVER_IMPL`: select volume-server implementation (`go` default, `rust` for a Rust volume server).
|
||||
- `RUST_VOLUME_BINARY`: explicit path to the Rust volume-server executable (overrides crate auto-detect).
|
||||
- `VOLUME_SERVER_RUST_MODE`: `weed-volume-rs` launcher mode (`native` default, `exec` for direct Go delegation, `proxy` for Rust front proxy + Go backend).
|
||||
- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
|
||||
|
||||
## Current scope (Phase 0)
|
||||
@@ -27,4 +24,4 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
- Initial HTTP admin endpoint checks
|
||||
- Initial gRPC state/status checks
|
||||
|
||||
More API coverage is tracked in `test/volume_server/DEV_PLAN.md`.
|
||||
More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.
|
||||
@@ -287,9 +287,6 @@ func (c *Cluster) startVolume(dataDirs []string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
c.volumeCmd = exec.Command(c.weedBinary, args...)
|
||||
c.volumeCmd.Dir = c.baseDir
|
||||
|
||||
@@ -265,9 +265,6 @@ func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
|
||||
@@ -267,9 +267,6 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
|
||||
@@ -214,9 +214,6 @@ func rustVolumeArgs(
|
||||
if profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "--inflightDownloadDataTimeout", profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "--fileSizeLimitMB", strconv.Itoa(profile.FileSizeLimitMB))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -255,21 +252,12 @@ func FindOrBuildRustBinary() (string, error) {
|
||||
rustBinaryOnce.Do(func() {
|
||||
// Derive the Rust volume crate directory from this source file's location.
|
||||
rustCrateDir := ""
|
||||
releaseBin := ""
|
||||
if _, file, _, ok := runtime.Caller(0); ok {
|
||||
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
|
||||
for _, candidate := range []struct {
|
||||
dir string
|
||||
bin string
|
||||
}{
|
||||
{"seaweed-volume", "weed-volume"},
|
||||
{"weed-volume", "weed-volume"},
|
||||
{filepath.Join("rust", "volume_server"), "weed-volume-rs"},
|
||||
} {
|
||||
dir := filepath.Join(repoRoot, candidate.dir)
|
||||
for _, candidate := range []string{"seaweed-volume", "weed-volume"} {
|
||||
dir := filepath.Join(repoRoot, candidate)
|
||||
if isDir(dir) && isFile(filepath.Join(dir, "Cargo.toml")) {
|
||||
rustCrateDir = dir
|
||||
releaseBin = filepath.Join(dir, "target", "release", candidate.bin)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -279,6 +267,8 @@ func FindOrBuildRustBinary() (string, error) {
|
||||
return
|
||||
}
|
||||
|
||||
releaseBin := filepath.Join(rustCrateDir, "target", "release", "weed-volume")
|
||||
|
||||
// Always rebuild once per test process so the harness uses current source
|
||||
// and features. The crate defaults to 5bytes, so a test binary built
|
||||
// without 5BytesOffset has to turn it off or the Rust server refuses the
|
||||
|
||||
@@ -16,39 +16,31 @@ import (
|
||||
|
||||
func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, "", "")
|
||||
}
|
||||
|
||||
func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection, replication string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, replication, "")
|
||||
allocateVolume(t, client, volumeID, collection, "")
|
||||
}
|
||||
|
||||
// AllocateVolumeWithTTL allocates a volume whose needles expire after ttl
|
||||
// (e.g. "10m").
|
||||
func AllocateVolumeWithTTL(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, ttl string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, "", ttl)
|
||||
allocateVolume(t, client, volumeID, collection, ttl)
|
||||
}
|
||||
|
||||
func allocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, replication, ttl string) {
|
||||
func allocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, ttl string) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if replication == "" {
|
||||
replication = "000"
|
||||
}
|
||||
|
||||
_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
Replication: "000",
|
||||
Ttl: ttl,
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume %d (replication=%s): %v", volumeID, replication, err)
|
||||
t.Fatalf("allocate volume %d: %v", volumeID, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -456,19 +456,6 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping filer expected InvalidArgument, got %s: %v", got, err)
|
||||
}
|
||||
|
||||
// Volume-server targets are not tracked peers either, so they are
|
||||
// rejected at admission the same way filer targets are.
|
||||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.VolumeServerType,
|
||||
Target: "127.0.0.1:1.1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping volume server target should be rejected by admission")
|
||||
}
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping volume server expected InvalidArgument, got %s: %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPingMasterTargetSuccess(t *testing.T) {
|
||||
|
||||
@@ -3,10 +3,7 @@ package volume_server_grpc_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -268,141 +265,3 @@ func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
|
||||
t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchDeleteEcNeedleSuccess(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
if stopProxy := maybeStartGrpcOffsetProxy(t, cluster.VolumeAdminAddress(), cluster.VolumeGRPCAddress()); stopProxy != nil {
|
||||
t.Cleanup(stopProxy)
|
||||
}
|
||||
|
||||
const volumeID = uint32(34)
|
||||
const needleID = uint64(930001)
|
||||
const cookie = uint32(0x6677BBCC)
|
||||
framework.AllocateVolume(t, client, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-ec-success"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
deleteResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
|
||||
if err != nil {
|
||||
t.Fatalf("BatchDelete EC needle should return response, got grpc error: %v", err)
|
||||
}
|
||||
if len(deleteResp.GetResults()) != 1 {
|
||||
t.Fatalf("BatchDelete EC needle expected one result, got %d", len(deleteResp.GetResults()))
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetStatus() != http.StatusAccepted {
|
||||
t.Fatalf("BatchDelete EC needle expected status 202, got %d error=%q", deleteResp.GetResults()[0].GetStatus(), deleteResp.GetResults()[0].GetError())
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetSize() == 0 {
|
||||
t.Fatalf("BatchDelete EC needle expected non-zero deleted size")
|
||||
}
|
||||
|
||||
deletedStream, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
|
||||
VolumeId: volumeID,
|
||||
ShardId: 0,
|
||||
FileKey: needleID,
|
||||
Offset: 0,
|
||||
Size: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
|
||||
}
|
||||
deletedMsg, err := deletedStream.Recv()
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
|
||||
}
|
||||
if !deletedMsg.GetIsDeleted() {
|
||||
t.Fatalf("VolumeEcShardRead expected IsDeleted=true after EC batch delete")
|
||||
}
|
||||
}
|
||||
|
||||
func maybeStartGrpcOffsetProxy(t testing.TB, adminAddr, actualGrpcAddr string) func() {
|
||||
t.Helper()
|
||||
|
||||
host, portText, err := net.SplitHostPort(adminAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("split admin address %q: %v", adminAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portText)
|
||||
if err != nil {
|
||||
t.Fatalf("parse admin port %q: %v", portText, err)
|
||||
}
|
||||
|
||||
expectedGrpcAddr := net.JoinHostPort(host, strconv.Itoa(port+10000))
|
||||
if expectedGrpcAddr == actualGrpcAddr {
|
||||
return nil
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", expectedGrpcAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("listen grpc-offset proxy %s -> %s: %v", expectedGrpcAddr, actualGrpcAddr, err)
|
||||
}
|
||||
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
conn, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
select {
|
||||
case <-stopped:
|
||||
return
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
go proxyBidirectional(conn, actualGrpcAddr)
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
close(stopped)
|
||||
_ = listener.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func proxyBidirectional(inbound net.Conn, targetAddr string) {
|
||||
outbound, err := net.Dial("tcp", targetAddr)
|
||||
if err != nil {
|
||||
_ = inbound.Close()
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
_, _ = io.Copy(outbound, inbound)
|
||||
_ = outbound.Close()
|
||||
}()
|
||||
_, _ = io.Copy(inbound, outbound)
|
||||
_ = inbound.Close()
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -145,66 +144,3 @@ func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
|
||||
t.Fatalf("ReadNeedleMeta should fail for invalid offset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadNeedleMetaUnsupportedForEcMountedVolume(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(93)
|
||||
const needleID = uint64(880002)
|
||||
const cookie = uint32(0x11AA22BB)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-meta-unsupported"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeUnmount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: needleID,
|
||||
Offset: 0,
|
||||
Size: 128,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("ReadNeedleMeta should fail when only EC shards are mounted")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "ec shards is not supported") {
|
||||
t.Fatalf("ReadNeedleMeta EC-only error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -426,67 +426,3 @@ func findLastNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64)
|
||||
}
|
||||
return offset, size
|
||||
}
|
||||
func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeIDA = uint32(86)
|
||||
const volumeIDB = uint32(87)
|
||||
const needleIDA = uint64(445561)
|
||||
const needleIDB = uint64(445562)
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDA, "")
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDB, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fidA := framework.NewFileID(volumeIDA, needleIDA, 0xAA11CC22)
|
||||
fidB := framework.NewFileID(volumeIDB, needleIDB, 0xBB22DD33)
|
||||
payloadA := "read-all-multi-volume-a"
|
||||
payloadB := "read-all-multi-volume-b"
|
||||
|
||||
uploadA := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidA, []byte(payloadA))
|
||||
_ = framework.ReadAllAndClose(t, uploadA)
|
||||
if uploadA.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload A expected 201, got %d", uploadA.StatusCode)
|
||||
}
|
||||
uploadB := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidB, []byte(payloadB))
|
||||
_ = framework.ReadAllAndClose(t, uploadB)
|
||||
if uploadB.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload B expected 201, got %d", uploadB.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
|
||||
VolumeIds: []uint32{volumeIDA, volumeIDB},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAllNeedles start failed: %v", err)
|
||||
}
|
||||
|
||||
seen := map[uint64]string{}
|
||||
for {
|
||||
msg, recvErr := stream.Recv()
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if recvErr != nil {
|
||||
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
|
||||
}
|
||||
if msg.GetNeedleId() == needleIDA || msg.GetNeedleId() == needleIDB {
|
||||
seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
|
||||
}
|
||||
}
|
||||
|
||||
if got := seen[needleIDA]; got != payloadA {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume A needle: got %q want %q", got, payloadA)
|
||||
}
|
||||
if got := seen[needleIDB]; got != payloadB {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume B needle: got %q want %q", got, payloadB)
|
||||
}
|
||||
}
|
||||
@@ -280,54 +280,6 @@ func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryCsvInputWithCsvPayloadStillReturnsNoRows(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(67)
|
||||
const needleID = uint64(777004)
|
||||
const cookie = uint32(0xDCDCADAD)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
csvPayload := []byte("name,score\nalice,12\nbob,3\n")
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, csvPayload)
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != 201 {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
|
||||
FromFileIds: []string{fid},
|
||||
Selections: []string{"score"},
|
||||
Filter: &volume_server_pb.QueryRequest_Filter{
|
||||
Field: "score",
|
||||
Operand: ">",
|
||||
Value: "10",
|
||||
},
|
||||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
|
||||
CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query csv start failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = csvStream.Recv()
|
||||
if err != io.EOF {
|
||||
t.Fatalf("Query csv with csv payload expected EOF with no rows, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
|
||||
@@ -12,8 +12,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestVolumeTailSenderMissingVolume(t *testing.T) {
|
||||
@@ -93,36 +91,6 @@ func TestVolumeTailReceiverMissingVolume(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverSourceUnavailable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(89)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceVolumeServer: "127.0.0.1:19999.29999",
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 1,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailReceiver should fail when source volume server is unavailable")
|
||||
}
|
||||
lowered := strings.ToLower(err.Error())
|
||||
if !strings.Contains(lowered, "dial") && !strings.Contains(lowered, "unavailable") && !strings.Contains(lowered, "connection refused") {
|
||||
t.Fatalf("VolumeTailReceiver source-unavailable error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
@@ -236,46 +204,3 @@ func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
|
||||
t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailSenderStreamCancellation(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(74)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 30,
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender start failed: %v", err)
|
||||
}
|
||||
|
||||
firstMsg, err := stream.Recv()
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender first recv failed: %v", err)
|
||||
}
|
||||
if !firstMsg.GetIsLastChunk() {
|
||||
cancel()
|
||||
t.Fatalf("expected heartbeat first message before cancellation")
|
||||
}
|
||||
|
||||
cancel()
|
||||
_, err = stream.Recv()
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailSender recv after cancellation expected error")
|
||||
}
|
||||
if status.Code(err) != codes.Canceled {
|
||||
t.Fatalf("VolumeTailSender cancellation expected grpc code Canceled, got %v (%v)", status.Code(err), err)
|
||||
}
|
||||
}
|
||||
@@ -174,26 +174,6 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P3()
|
||||
profile.EnableUIAccess = true
|
||||
cluster := framework.StartSingleVolumeCluster(t, profile)
|
||||
client := framework.NewHTTPClient()
|
||||
|
||||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected /ui/index.html to be exposed when access.ui=true under JWT profile, got %d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(body)), "volume") {
|
||||
t.Fatalf("ui page does not look like volume status page")
|
||||
}
|
||||
}
|
||||
|
||||
func mustNewRequest(t testing.TB, method, url string) *http.Request {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
|
||||
@@ -241,90 +241,3 @@ func TestPublicPortHeadReadParity(t *testing.T) {
|
||||
t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
|
||||
}
|
||||
}
|
||||
func TestUnsupportedMethodConnectParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(85)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 124001, 0x03030303)
|
||||
client := framework.NewHTTPClient()
|
||||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
adminResp := framework.DoRequest(t, client, adminReq)
|
||||
_ = framework.ReadAllAndClose(t, adminResp)
|
||||
if adminResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode)
|
||||
}
|
||||
|
||||
publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid)
|
||||
publicResp := framework.DoRequest(t, client, publicReq)
|
||||
_ = framework.ReadAllAndClose(t, publicResp)
|
||||
if publicResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode)
|
||||
}
|
||||
|
||||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
verifyBody := framework.ReadAllAndClose(t, verifyResp)
|
||||
if verifyResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
|
||||
}
|
||||
if string(verifyBody) != "connect-method-check" {
|
||||
t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedMethodMkcolParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(88)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 124004, 0x06060606)
|
||||
client := framework.NewHTTPClient()
|
||||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("mkcol-method-check"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
adminReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
adminResp := framework.DoRequest(t, client, adminReq)
|
||||
_ = framework.ReadAllAndClose(t, adminResp)
|
||||
if adminResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("admin MKCOL expected 400, got %d", adminResp.StatusCode)
|
||||
}
|
||||
|
||||
publicReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumePublicURL()+"/"+fid)
|
||||
publicResp := framework.DoRequest(t, client, publicReq)
|
||||
_ = framework.ReadAllAndClose(t, publicResp)
|
||||
if publicResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("public MKCOL expected passthrough 200, got %d", publicResp.StatusCode)
|
||||
}
|
||||
|
||||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
verifyBody := framework.ReadAllAndClose(t, verifyResp)
|
||||
if verifyResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
|
||||
}
|
||||
if string(verifyBody) != "mkcol-method-check" {
|
||||
t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package volume_server_http_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
@@ -52,17 +51,4 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
|
||||
if string(readDeletedBody) != string(payload) {
|
||||
t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
|
||||
}
|
||||
|
||||
headReadDeletedReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
|
||||
headReadDeletedResp := framework.DoRequest(t, client, headReadDeletedReq)
|
||||
headReadDeletedBody := framework.ReadAllAndClose(t, headReadDeletedResp)
|
||||
if headReadDeletedResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD with readDeleted=true expected 200, got %d", headReadDeletedResp.StatusCode)
|
||||
}
|
||||
if len(headReadDeletedBody) != 0 {
|
||||
t.Fatalf("HEAD with readDeleted=true expected empty body, got %d bytes", len(headReadDeletedBody))
|
||||
}
|
||||
if got := headReadDeletedResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
|
||||
t.Fatalf("HEAD with readDeleted=true content-length mismatch: got %q want %d", got, len(payload))
|
||||
}
|
||||
}
|
||||
@@ -164,10 +164,6 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if lastModified == "" {
|
||||
t.Fatalf("baseline read expected Last-Modified header")
|
||||
}
|
||||
etag := baselineResp.Header.Get("ETag")
|
||||
if etag == "" {
|
||||
t.Fatalf("baseline read expected ETag header")
|
||||
}
|
||||
|
||||
precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
precedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
@@ -192,28 +188,4 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if string(invalidIMSBody) != string(payload) {
|
||||
t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
|
||||
}
|
||||
|
||||
headIMSPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headIMSPrecedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
headIMSPrecedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
|
||||
headIMSPrecedenceResp := framework.DoRequest(t, client, headIMSPrecedenceReq)
|
||||
headIMSPrecedenceBody := framework.ReadAllAndClose(t, headIMSPrecedenceResp)
|
||||
if headIMSPrecedenceResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD mismatched If-None-Match overrides If-Modified-Since: expected 200, got %d", headIMSPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headIMSPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional precedence expected empty body, got %d bytes", len(headIMSPrecedenceBody))
|
||||
}
|
||||
|
||||
headETagPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headETagPrecedenceReq.Header.Set("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT")
|
||||
headETagPrecedenceReq.Header.Set("If-None-Match", etag)
|
||||
headETagPrecedenceResp := framework.DoRequest(t, client, headETagPrecedenceReq)
|
||||
headETagPrecedenceBody := framework.ReadAllAndClose(t, headETagPrecedenceResp)
|
||||
if headETagPrecedenceResp.StatusCode != http.StatusNotModified {
|
||||
t.Fatalf("HEAD conditional precedence expected 304 when If-None-Match matches, got %d", headETagPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headETagPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional etag precedence expected empty body, got %d bytes", len(headETagPrecedenceBody))
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package volume_server_http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -73,64 +72,3 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
|
||||
t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRejectsPayloadOverFileSizeLimit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P1()
|
||||
profile.FileSizeLimitMB = 1
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(99)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772002, 0x2A3B4C5D)
|
||||
oversizedPayload := bytes.Repeat([]byte("z"), 1024*1024+1)
|
||||
|
||||
oversizedReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, oversizedPayload)
|
||||
oversizedResp := framework.DoRequest(t, client, oversizedReq)
|
||||
oversizedBody := framework.ReadAllAndClose(t, oversizedResp)
|
||||
if oversizedResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("oversized write expected 400, got %d", oversizedResp.StatusCode)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(oversizedBody)), "limited") {
|
||||
t.Fatalf("oversized write response should mention limit, got %q", string(oversizedBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicatedWriteFailsWhenReplicaRequirementsNotMet(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(109)
|
||||
framework.AllocateVolumeWithReplication(t, grpcClient, volumeID, "", "001")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772003, 0x3A4B5C6D)
|
||||
payload := []byte("replicated-write-failure-path")
|
||||
|
||||
writeResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
|
||||
writeBody := framework.ReadAllAndClose(t, writeResp)
|
||||
if writeResp.StatusCode != http.StatusInternalServerError {
|
||||
t.Fatalf("replicated write with unmet replication requirements expected 500, got %d body=%s", writeResp.StatusCode, string(writeBody))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(writeBody)), "replica") {
|
||||
t.Fatalf("replicated write failure response should mention replica write failure, got %q", string(writeBody))
|
||||
}
|
||||
|
||||
readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
_ = framework.ReadAllAndClose(t, readResp)
|
||||
if readResp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("local read after failed replicate write expected 404 (not committed locally), got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,6 @@ type Profile struct {
|
||||
ConcurrentDownloadLimitMB int
|
||||
InflightUploadTimeout time.Duration
|
||||
InflightDownloadTimeout time.Duration
|
||||
FileSizeLimitMB int
|
||||
|
||||
ReplicatedLayout bool
|
||||
HasErasureCoding bool
|
||||
|
||||
@@ -111,6 +111,16 @@ func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error
|
||||
}
|
||||
ecChanged = true
|
||||
}
|
||||
if k := "maintenance.erasure_coding.max_shard_transfers"; v.IsSet(k) {
|
||||
n := v.GetInt(k)
|
||||
if n < 0 {
|
||||
n = 0
|
||||
} else if n > math.MaxInt32 {
|
||||
n = math.MaxInt32
|
||||
}
|
||||
ecConf.MaxShardTransfers = &n
|
||||
ecChanged = true
|
||||
}
|
||||
|
||||
if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged {
|
||||
return nil
|
||||
@@ -239,6 +249,7 @@ var pluginConfigSections = []pluginConfigSection{
|
||||
"replica_placement": stringValue,
|
||||
"strict_placement": boolValue,
|
||||
"free_slots_deadband": int64Value,
|
||||
"max_shard_transfers": int64Value,
|
||||
},
|
||||
// workers read collection_filter from the admin values, not the worker values
|
||||
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
|
||||
|
||||
@@ -307,3 +307,97 @@ preferred_tags = ["fast,SSD"]
|
||||
t.Errorf("preferred_tags = %v, want [fast ssd]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMaintenanceConfigFromTomlMaxShardTransfers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cp := NewConfigPersistence(dir)
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
fullness_ratio = 0.9
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply fullness: %v", err)
|
||||
}
|
||||
ec := erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers != nil {
|
||||
t.Fatalf("absent key persisted as %d, want unset", *ec.MaxShardTransfers)
|
||||
}
|
||||
if ec.FullnessRatio != 0.9 {
|
||||
t.Fatalf("fullness_ratio = %v, want 0.9", ec.FullnessRatio)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 0
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 0: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 0 {
|
||||
t.Fatalf("0: got %v, want sequential 0", ec.MaxShardTransfers)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 4
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 4: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 4 {
|
||||
t.Fatalf("4: got %v, want 4", ec.MaxShardTransfers)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = -3
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply negative: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 0 {
|
||||
t.Fatalf("negative: got %v, want sequential 0", ec.MaxShardTransfers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPluginConfigFromTomlMaxShardTransfers(t *testing.T) {
|
||||
s := newPluginServer(t)
|
||||
seed := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "erasure_coding",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
|
||||
}
|
||||
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
if err := s.ApplyPluginConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
replica_placement = "020"
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply placement: %v", err)
|
||||
}
|
||||
cfg, err := s.GetPlugin().LoadJobTypeConfig("erasure_coding")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load: %v %v", cfg, err)
|
||||
}
|
||||
if _, ok := cfg.WorkerConfigValues["max_shard_transfers"]; ok {
|
||||
t.Fatal("absent max_shard_transfers was written into worker config")
|
||||
}
|
||||
|
||||
if err := s.ApplyPluginConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 0
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 0: %v", err)
|
||||
}
|
||||
cfg, err = s.GetPlugin().LoadJobTypeConfig("erasure_coding")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load 0: %v %v", cfg, err)
|
||||
}
|
||||
got, ok := cfg.WorkerConfigValues["max_shard_transfers"]
|
||||
if !ok {
|
||||
t.Fatal("max_shard_transfers = 0 missing from worker config")
|
||||
}
|
||||
if got.GetInt64Value() != 0 {
|
||||
t.Fatalf("max_shard_transfers = %d, want 0", got.GetInt64Value())
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -50,9 +51,14 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers
|
||||
// ShowFileBrowser renders the file browser page
|
||||
func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
|
||||
// Get path from query parameter, default to root
|
||||
path := defaultQuery(r.URL.Query().Get("path"), "/")
|
||||
// Normalize Windows-style paths for consistency
|
||||
path = util.CleanWindowsPath(path)
|
||||
dirPath := path.Clean(util.CleanWindowsPath(defaultQuery(r.URL.Query().Get("path"), "/")))
|
||||
if !strings.HasPrefix(dirPath, "/") {
|
||||
dirPath = "/" + dirPath
|
||||
}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) && restrictedBrowserPath(dirPath) {
|
||||
http.Redirect(w, r, dash.P(r.Context(), "/admin?error=Insufficient permissions"), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
// Get pagination parameters
|
||||
lastFileName := r.URL.Query().Get("lastFileName")
|
||||
@@ -66,7 +72,7 @@ func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Get file browser data with cursor-based pagination
|
||||
browserData, err := h.adminServer.GetFileBrowser(path, "", lastFileName, pageSize)
|
||||
browserData, err := h.adminServer.GetFileBrowser(dirPath, "", lastFileName, pageSize)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get file browser data: "+err.Error())
|
||||
return
|
||||
@@ -356,14 +362,46 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
|
||||
cleanPath = "/" + cleanPath
|
||||
}
|
||||
|
||||
// Prevent path traversal attacks
|
||||
if strings.Contains(cleanPath, "..") {
|
||||
// Reject parent-directory components; names merely containing ".." are valid
|
||||
if slices.Contains(strings.Split(cleanPath, "/"), "..") {
|
||||
return "", fmt.Errorf("path traversal not allowed")
|
||||
}
|
||||
|
||||
return cleanPath, nil
|
||||
}
|
||||
|
||||
// restrictedBrowserPath reports whether a cleaned path lies inside the filer's
|
||||
// /etc tree, which holds the credential store and cluster configuration.
|
||||
func restrictedBrowserPath(cleanPath string) bool {
|
||||
return cleanPath == strings.TrimSuffix(filer.DirectoryEtcRoot, "/") ||
|
||||
strings.HasPrefix(cleanPath, filer.DirectoryEtcRoot)
|
||||
}
|
||||
|
||||
// cleanReadPath validates the request's path parameter; read-only sessions are
|
||||
// denied the /etc tree so the file browser cannot serve stored credentials.
|
||||
func (h *FileBrowserHandlers) cleanReadPath(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
cleanPath, err := h.validateAndCleanFilePath(r.URL.Query().Get("path"))
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return "", false
|
||||
}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) && restrictedBrowserPath(cleanPath) {
|
||||
writeJSONError(w, http.StatusForbidden, "Access to this path requires admin access")
|
||||
return "", false
|
||||
}
|
||||
return cleanPath, true
|
||||
}
|
||||
|
||||
// exportExcludedPrefixes lists the subtrees a metadata export skips; read-only
|
||||
// sessions also lose /etc so stored credentials stay out of the dump.
|
||||
func exportExcludedPrefixes(r *http.Request) []string {
|
||||
prefixes := []string{filer.SystemLogDir}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) {
|
||||
prefixes = append(prefixes, strings.TrimSuffix(filer.DirectoryEtcRoot, "/"))
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
|
||||
// fetchFileContent fetches file content via the filer gRPC service. It is
|
||||
// used for the "view as text" path, so the maxBytes cap matches the 1 MB
|
||||
// limit the caller already applies before invoking us.
|
||||
@@ -377,9 +415,8 @@ func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Dur
|
||||
// gRPC service, so the admin file browser keeps working even when the filer
|
||||
// is started with -disableHttp=true.
|
||||
func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
inline := r.URL.Query().Get("inline") == "true"
|
||||
@@ -419,9 +456,8 @@ func (t *responseWriteTracker) Write(p []byte) (int, error) {
|
||||
|
||||
// ViewFile handles file viewing requests (for text files, images, etc.)
|
||||
func (h *FileBrowserHandlers) ViewFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -524,9 +560,8 @@ func (h *FileBrowserHandlers) ViewFile(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// GetFileProperties handles file properties requests
|
||||
func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -630,23 +665,18 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
|
||||
// FullEntry stream — the format weed shell fs.meta.load reads. Directories are
|
||||
// walked recursively via the filer BFS metadata stream.
|
||||
func (h *FileBrowserHandlers) ExportMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
cleanPath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
cleanPath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
excludedPrefixes := exportExcludedPrefixes(r)
|
||||
|
||||
tracker := &responseWriteTracker{ResponseWriter: w}
|
||||
|
||||
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
stream, err := client.TraverseBfsMetadata(r.Context(), &filer_pb.TraverseBfsMetadataRequest{
|
||||
Directory: cleanPath,
|
||||
ExcludedPrefixes: []string{filer.SystemLogDir},
|
||||
ExcludedPrefixes: excludedPrefixes,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -3,9 +3,12 @@ package handlers
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
func TestValidateAndCleanFilePath_AllowsControlChars(t *testing.T) {
|
||||
@@ -26,6 +29,9 @@ func TestValidateAndCleanFilePath_AllowsControlChars(t *testing.T) {
|
||||
{"relative/path.txt", "/relative/path.txt"},
|
||||
// Duplicate slashes should be collapsed by path.Clean.
|
||||
{"/a//b", "/a/b"},
|
||||
// ".." inside a file or directory name is not a parent component.
|
||||
{"/buckets/reports/report..csv", "/buckets/reports/report..csv"},
|
||||
{"/a../b", "/a../b"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := h.validateAndCleanFilePath(tc.in)
|
||||
@@ -46,6 +52,81 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAndCleanFilePath_RejectsParentComponents(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
for _, p := range []string{"../etc", "../../etc/iam", "..", "a/../../b"} {
|
||||
if _, err := h.validateAndCleanFilePath(p); err == nil {
|
||||
t.Errorf("validateAndCleanFilePath(%q): expected traversal rejection", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_DeniesEtcForReadOnly(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
denied := []string{"/etc", "/etc/iam", "/etc/iam/identities/root.json",
|
||||
"/etc/s3/kek.json", "/etc/../etc/iam/identity.json", "/a/../../etc/x"}
|
||||
for _, p := range denied {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path="+p, nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); ok {
|
||||
t.Errorf("path %q: expected denial for read-only session", p)
|
||||
continue
|
||||
}
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Errorf("path %q: expected 403, got %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_AllowsEtcForAdminAndNorRole(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
for _, role := range []string{"admin", ""} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path=/etc/iam/identities/root.json", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "u", role, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); !ok {
|
||||
t.Errorf("role %q: expected /etc path to be allowed, got %d", role, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_AllowsReadOnlyOutsideEtc(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
for _, p := range []string{"/buckets/foo/bar.txt", "/etcx/file", "/etcetera", "/"} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path="+p, nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); !ok {
|
||||
t.Errorf("path %q: expected allowed for read-only session, got %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportExcludedPrefixes_ReadOnlyDropsEtc(t *testing.T) {
|
||||
etcPrefix := strings.TrimSuffix(filer.DirectoryEtcRoot, "/")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/metadata/export?path=/", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
if got := exportExcludedPrefixes(req); !slices.Contains(got, etcPrefix) {
|
||||
t.Errorf("read-only export should exclude %q, got %v", etcPrefix, got)
|
||||
}
|
||||
|
||||
for _, role := range []string{"admin", ""} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/metadata/export?path=/", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "u", role, "csrf"))
|
||||
if got := exportExcludedPrefixes(req); slices.Contains(got, etcPrefix) {
|
||||
t.Errorf("role %q: export should keep %q, got %v", role, etcPrefix, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
|
||||
h := &FileBrowserHandlers{adminServer: &dash.AdminServer{}}
|
||||
|
||||
@@ -62,4 +143,3 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2272,6 +2272,105 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
return input;
|
||||
}
|
||||
|
||||
function shardTransferParallelOn(nativeValue) {
|
||||
if (nativeValue === null || nativeValue === undefined || nativeValue === '') {
|
||||
return true;
|
||||
}
|
||||
var n = parseInt(nativeValue, 10);
|
||||
if (Number.isNaN(n)) {
|
||||
return true;
|
||||
}
|
||||
return n !== 0;
|
||||
}
|
||||
|
||||
function shardTransferMaxText(nativeValue) {
|
||||
if (nativeValue === null || nativeValue === undefined || nativeValue === '') {
|
||||
return '';
|
||||
}
|
||||
var n = parseInt(nativeValue, 10);
|
||||
if (Number.isNaN(n) || n <= 0) {
|
||||
return '';
|
||||
}
|
||||
return String(n);
|
||||
}
|
||||
|
||||
// Presentation of the single persisted max_shard_transfers value.
|
||||
// Unchecked stores 0. Checked with an empty max omits the key.
|
||||
function renderShardTransferControls(scope, field, nativeValue) {
|
||||
var parallelOn = shardTransferParallelOn(nativeValue);
|
||||
var wrapper = document.createElement('div');
|
||||
wrapper.className = 'mb-3 plugin-field';
|
||||
wrapper.dataset.scope = scope;
|
||||
wrapper.dataset.fieldName = field.name;
|
||||
wrapper.dataset.fieldType = normalizeEnum(field.field_type);
|
||||
wrapper.dataset.widget = 'SHARD_TRANSFERS';
|
||||
|
||||
var checkWrap = document.createElement('div');
|
||||
checkWrap.className = 'form-check form-switch mb-2';
|
||||
var box = document.createElement('input');
|
||||
box.type = 'checkbox';
|
||||
box.className = 'form-check-input';
|
||||
box.id = toInputID(scope, field.name) + '-parallel';
|
||||
box.checked = parallelOn;
|
||||
var boxLabel = document.createElement('label');
|
||||
boxLabel.className = 'form-check-label';
|
||||
boxLabel.setAttribute('for', box.id);
|
||||
boxLabel.textContent = 'Parallel shard distribution';
|
||||
checkWrap.appendChild(box);
|
||||
checkWrap.appendChild(boxLabel);
|
||||
wrapper.appendChild(checkWrap);
|
||||
|
||||
var numberLabel = document.createElement('label');
|
||||
numberLabel.className = 'form-label';
|
||||
numberLabel.setAttribute('for', toInputID(scope, field.name));
|
||||
numberLabel.textContent = 'Max transfers';
|
||||
wrapper.appendChild(numberLabel);
|
||||
|
||||
var input = document.createElement('input');
|
||||
input.type = 'number';
|
||||
input.min = '1';
|
||||
input.step = '1';
|
||||
input.className = 'form-control';
|
||||
input.id = toInputID(scope, field.name);
|
||||
input.placeholder = 'no global cap';
|
||||
input.value = shardTransferMaxText(nativeValue);
|
||||
input.disabled = !parallelOn;
|
||||
wrapper.appendChild(input);
|
||||
|
||||
box.addEventListener('change', function() {
|
||||
input.disabled = !box.checked;
|
||||
});
|
||||
|
||||
var help = document.createElement('div');
|
||||
help.className = 'form-text';
|
||||
help.textContent = field.description || 'Unchecked sends shards sequentially. Checked with an empty max uses one in-flight copy per destination disk. A number is a global cap, still one copy per disk.';
|
||||
wrapper.appendChild(help);
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
function collectShardTransferValue(scope) {
|
||||
var box = document.getElementById(toInputID(scope, 'max_shard_transfers') + '-parallel');
|
||||
var input = document.getElementById(toInputID(scope, 'max_shard_transfers'));
|
||||
if (!box || !input) {
|
||||
return null;
|
||||
}
|
||||
if (!box.checked) {
|
||||
return { int64_value: '0' };
|
||||
}
|
||||
var raw = String(input.value || '').trim();
|
||||
if (raw === '') {
|
||||
return null;
|
||||
}
|
||||
if (!/^\d+$/.test(raw)) {
|
||||
throw new Error('Invalid int64 value for field Max transfers');
|
||||
}
|
||||
var n = parseInt(raw, 10);
|
||||
if (n <= 0) {
|
||||
return { int64_value: '0' };
|
||||
}
|
||||
return { int64_value: String(n) };
|
||||
}
|
||||
|
||||
function renderConfigForm(scope, rootID, form) {
|
||||
var root = document.getElementById(rootID);
|
||||
if (!root) {
|
||||
@@ -2321,6 +2420,11 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
var configValue = resolveFieldConfigValue(scope, field);
|
||||
var nativeValue = configValueToNative(configValue);
|
||||
|
||||
if (field.name === 'max_shard_transfers') {
|
||||
sectionBody.appendChild(renderShardTransferControls(scope, field, nativeValue));
|
||||
continue;
|
||||
}
|
||||
|
||||
var wrapper = document.createElement('div');
|
||||
wrapper.className = 'mb-3 plugin-field';
|
||||
wrapper.dataset.scope = scope;
|
||||
@@ -2440,6 +2544,14 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.name === 'max_shard_transfers') {
|
||||
var shardCap = collectShardTransferValue(scope);
|
||||
if (shardCap !== null) {
|
||||
out[field.name] = shardCap;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var wrapper = document.querySelector('.plugin-field[data-scope="' + scope + '"][data-field-name="' + field.name + '"]');
|
||||
if (wrapper && wrapper.classList.contains('plugin-field-hidden')) {
|
||||
continue;
|
||||
|
||||
@@ -104,7 +104,7 @@ func Plugin(page string, initialJob string, lane string) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"bi bi-arrow-repeat me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"bi bi-graph-up-arrow me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"bi bi-sliders me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"bi bi-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"bi bi-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"bi bi-list-check me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-3 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-secondary h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Next Run</div><h4 class=\"mb-0 plugin-scheduler-next-run\">-</h4><div class=\"text-muted small plugin-scheduler-next-run-meta\"></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-bar-chart me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"bi bi-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Sequential scheduler with per-job runtime limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Max Runtime</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th><th>Last Run</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-people me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"bi bi-sliders me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-body\"><div class=Line truncated
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"bi bi-arrow-repeat me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"bi bi-graph-up-arrow me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"bi bi-sliders me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"bi bi-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"bi bi-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"bi bi-list-check me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-3 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-secondary h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Next Run</div><h4 class=\"mb-0 plugin-scheduler-next-run\">-</h4><div class=\"text-muted small plugin-scheduler-next-run-meta\"></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-bar-chart me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"bi bi-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Sequential scheduler with per-job runtime limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Max Runtime</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th><th>Last Run</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-people me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"bi bi-sliders me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-body\"><div class=Line truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
+18
-3
@@ -64,6 +64,7 @@ type FilerOptions struct {
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
portGrpc *int
|
||||
publicPort *int
|
||||
@@ -282,12 +283,17 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
}
|
||||
go stats_collect.StartMetricsServer(*f.metricsHttpIp, *f.metricsHttpPort)
|
||||
|
||||
// Embedded services keep their loopback companion listener only while
|
||||
// -ip.bind is unset and the bind address falls back to -ip. This must be
|
||||
// captured before startFiler fills *f.bindIp in place.
|
||||
filerBindIpUnset := *f.bindIp == ""
|
||||
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
filerS3Options.ip = f.ip
|
||||
if *filerS3Options.bindIp == "" {
|
||||
filerS3Options.listenOnLocalhost = &filerBindIpUnset
|
||||
filerS3Options.bindIp = f.bindIp
|
||||
}
|
||||
filerS3Options.localFilerSocket = f.localSocket
|
||||
@@ -320,6 +326,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
if *filerStartIam {
|
||||
filerIamOptions.filer = &filerAddress
|
||||
filerIamOptions.masters = f.mastersString
|
||||
filerIamOptions.listenOnLocalhost = &filerBindIpUnset
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerIamOptions.startIamServer()
|
||||
@@ -330,6 +337,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
if *filerStartSftp {
|
||||
filerSftpOptions.filer = &filerAddress
|
||||
if *filerSftpOptions.bindIp == "" {
|
||||
filerSftpOptions.listenOnLocalhost = &filerBindIpUnset
|
||||
filerSftpOptions.bindIp = f.bindIp
|
||||
}
|
||||
go func(delay time.Duration) {
|
||||
@@ -356,6 +364,13 @@ func (fo *FilerOptions) startFiler() {
|
||||
if *fo.portGrpc == 0 {
|
||||
*fo.portGrpc = 10000 + *fo.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset
|
||||
// and the bind address falls back to -ip. An explicit -ip.bind pins each
|
||||
// port to exactly that address.
|
||||
listenOnLocalhost := *fo.bindIp == ""
|
||||
if fo.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *fo.listenOnLocalhost
|
||||
}
|
||||
if *fo.bindIp == "" {
|
||||
*fo.bindIp = *fo.ip
|
||||
}
|
||||
@@ -433,7 +448,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
if *fo.publicPort != 0 {
|
||||
publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
|
||||
glog.V(0).Infoln("Start Seaweed filer server", version.Version(), "public at", publicListeningAddress)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
|
||||
}
|
||||
@@ -459,7 +474,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", version.Version(), *fo.ip, *fo.port)
|
||||
filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
|
||||
*fo.bindIp, *fo.port,
|
||||
time.Duration(10)*time.Second,
|
||||
time.Duration(10)*time.Second, listenOnLocalhost,
|
||||
)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer listener error: %v", e)
|
||||
@@ -467,7 +482,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *fo.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
|
||||
+16
-1
@@ -31,6 +31,8 @@ type IamOptions struct {
|
||||
filer *string
|
||||
masters *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
selfServiceRequiresGrant *bool
|
||||
}
|
||||
@@ -40,6 +42,7 @@ func init() {
|
||||
iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
|
||||
iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
|
||||
iamStandaloneOptions.ipBind = cmdIam.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
|
||||
iamStandaloneOptions.selfServiceRequiresGrant = cmdIam.Flag.Bool("selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
}
|
||||
@@ -119,8 +122,20 @@ func (iamopt *IamOptions) startIamServer() bool {
|
||||
iamApiServer.Shutdown()
|
||||
})
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset and
|
||||
// the bind address falls back to -ip. An explicit -ip.bind pins the port to
|
||||
// exactly that address.
|
||||
bindIp := *iamopt.ip
|
||||
listenOnLocalhost := iamopt.ipBind == nil || *iamopt.ipBind == ""
|
||||
if iamopt.ipBind != nil && *iamopt.ipBind != "" {
|
||||
bindIp = *iamopt.ipBind
|
||||
}
|
||||
if iamopt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *iamopt.listenOnLocalhost
|
||||
}
|
||||
|
||||
listenAddress := fmt.Sprintf(":%d", *iamopt.port)
|
||||
iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
|
||||
iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(bindIp, *iamopt.port, time.Duration(10)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
|
||||
+30
-11
@@ -50,6 +50,7 @@ type MasterOptions struct {
|
||||
portGrpc *int
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
metaFolder *string
|
||||
peers *string
|
||||
mastersDeprecated *string // deprecated, for backward compatibility in master.follower
|
||||
@@ -81,6 +82,15 @@ type MasterOptions struct {
|
||||
// is cancelled. Used by integration tests and by weed mini; nil for
|
||||
// standalone weed master.
|
||||
shutdownCtx context.Context
|
||||
// releaseAllocOnFirstReport lifts the post-mint allocation hold as soon as
|
||||
// one full volume-server inventory has been applied. weed mini sets it in
|
||||
// single-master mode, with singleReporterAddr pinning the embedded volume
|
||||
// server as the only admitted reporter: any other volume server is refused,
|
||||
// so the warmup window after a fresh topology id is dead time.
|
||||
// Standalone weed master / weed server can face late-arriving veterans and
|
||||
// must keep the window.
|
||||
releaseAllocOnFirstReport bool
|
||||
singleReporterAddr string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -191,6 +201,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
if *masterOption.portGrpc == 0 {
|
||||
*masterOption.portGrpc = 10000 + *masterOption.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset
|
||||
// and the bind address falls back to -ip. An explicit -ip.bind pins each
|
||||
// port to exactly that address.
|
||||
listenOnLocalhost := *masterOption.ipBind == ""
|
||||
if masterOption.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *masterOption.listenOnLocalhost
|
||||
}
|
||||
if *masterOption.ipBind == "" {
|
||||
*masterOption.ipBind = *masterOption.ip
|
||||
}
|
||||
@@ -207,7 +224,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
|
||||
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Master startup error: %v", e)
|
||||
}
|
||||
@@ -264,7 +281,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
// starting grpc server
|
||||
grpcPort := *masterOption.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
@@ -480,14 +497,16 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
|
||||
VacuumIntervalSeconds: *m.vacuumIntervalSeconds,
|
||||
VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds,
|
||||
// PulseSeconds: *m.pulseSeconds,
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
ReleaseAllocOnFirstReport: m.releaseAllocOnFirstReport,
|
||||
SingleVolumeReporter: m.singleReporterAddr,
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,10 @@ func startMasterFollower(masterOptions MasterOptions) {
|
||||
option := masterOptions.toMasterOption(nil)
|
||||
option.IsFollower = true
|
||||
|
||||
listenOnLocalhost := *masterOptions.ipBind == ""
|
||||
if masterOptions.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *masterOptions.listenOnLocalhost
|
||||
}
|
||||
if *masterOptions.ipBind == "" {
|
||||
*masterOptions.ipBind = *masterOptions.ip
|
||||
}
|
||||
@@ -137,14 +141,14 @@ func startMasterFollower(masterOptions MasterOptions) {
|
||||
ms := weed_server.NewMasterServer(r, option, masters)
|
||||
listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Master startup error: %v", e)
|
||||
}
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *masterOptions.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
|
||||
+127
-65
@@ -821,10 +821,20 @@ func isPortAvailable(port int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// isPortUsableOnIP reports whether port can be bound on ip. While mini also
|
||||
// opens a loopback listener, the port must additionally be free on every
|
||||
// interface so that listener cannot collide with a holder on another address.
|
||||
func isPortUsableOnIP(ip string, port int, listenOnLocalhost bool) bool {
|
||||
if !isPortOpenOnIP(ip, port) {
|
||||
return false
|
||||
}
|
||||
return !listenOnLocalhost || isPortAvailable(port)
|
||||
}
|
||||
|
||||
// findAvailablePortOnIP finds the next available port on a specific IP starting from the given port
|
||||
// It skips any ports that are in the reservedPorts map (for gRPC port collision avoidance)
|
||||
// It returns the first available port found within maxAttempts, or 0 if none found
|
||||
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
|
||||
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool, listenOnLocalhost bool) int {
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
port := startPort + i
|
||||
if port > 65535 {
|
||||
@@ -836,7 +846,7 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
continue
|
||||
}
|
||||
// Check on both the specific IP and on all interfaces for maximum reliability
|
||||
if isPortOpenOnIP(ip, port) && isPortAvailable(port) {
|
||||
if isPortUsableOnIP(ip, port, listenOnLocalhost) {
|
||||
return port
|
||||
}
|
||||
}
|
||||
@@ -847,7 +857,7 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
// ensurePortAvailableOnIP ensures a port pointer points to an available port on a specific IP
|
||||
// If the port is not available, it finds the next available port and updates the pointer
|
||||
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string, listenOnLocalhost bool) error {
|
||||
// Check if this port was explicitly specified by the user (from CLI or env, before the config file was applied)
|
||||
isExplicitPort := explicitFlags[flagName]
|
||||
|
||||
@@ -863,7 +873,7 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
return fmt.Errorf("port %d for %s (specified by flag %s) is reserved for gRPC calculation and cannot be used", original, serviceName, flagName)
|
||||
}
|
||||
glog.Warningf("Port %d for %s is reserved for gRPC calculation, finding alternative...", original, serviceName)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
|
||||
} else {
|
||||
@@ -874,14 +884,14 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
}
|
||||
|
||||
// Check on both the specific IP and on all interfaces (0.0.0.0) for maximum reliability
|
||||
if !isPortOpenOnIP(ip, original) || !isPortAvailable(original) {
|
||||
if !isPortUsableOnIP(ip, original, listenOnLocalhost) {
|
||||
// If explicitly specified, fail immediately with the originally requested port
|
||||
if isExplicitPort {
|
||||
return fmt.Errorf("port %d for %s (specified by flag %s) is not available on %s and cannot be used", original, serviceName, flagName, ip)
|
||||
}
|
||||
// For default ports, try to find an alternative
|
||||
glog.Warningf("Port %d for %s is not available on %s, finding alternative port...", original, serviceName, ip)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
|
||||
} else {
|
||||
@@ -897,53 +907,61 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
// ensureAllPortsAvailableOnIP ensures all mini service ports are available on a specific IP
|
||||
// Returns an error if an explicitly specified port is unavailable.
|
||||
// This should be called before starting any services
|
||||
func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
|
||||
portConfigs := []struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{
|
||||
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc},
|
||||
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc},
|
||||
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc},
|
||||
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc, listenOnLocalhost},
|
||||
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc, listenOnLocalhost},
|
||||
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc, listenOnLocalhost},
|
||||
}
|
||||
if *miniEnableS3 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc, listenOnLocalhost})
|
||||
if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.portIceberg, "Iceberg", "s3.port.iceberg", nil})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.portIceberg, "Iceberg", "s3.port.iceberg", nil, listenOnLocalhost})
|
||||
}
|
||||
if miniS3Options.portLance != nil && *miniS3Options.portLance > 0 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.portLance, "Lance", "s3.port.lance", nil})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.portLance, "Lance", "s3.port.lance", nil, listenOnLocalhost})
|
||||
}
|
||||
}
|
||||
// WebDAV and the Admin UI bind only -ip.bind; they never open a loopback
|
||||
// companion listener, so ports held elsewhere cannot collide with them.
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil},
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false},
|
||||
struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
|
||||
|
||||
// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
|
||||
// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
|
||||
@@ -956,13 +974,20 @@ func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
}
|
||||
}
|
||||
|
||||
applyMiniAdminCredentialFallback(&miniAdminOptions)
|
||||
adminBindIP := miniAdminEffectiveBindIP(bindIp)
|
||||
|
||||
// Check all HTTP ports sequentially to avoid race conditions
|
||||
// Each port check and allocation must complete before the next one starts
|
||||
// to prevent multiple goroutines from claiming the same available port
|
||||
// Also avoid allocating ports that are reserved for gRPC calculation
|
||||
for _, config := range portConfigs {
|
||||
checkIP := bindIp
|
||||
if config.name == "Admin" {
|
||||
checkIP = adminBindIP
|
||||
}
|
||||
original := *config.port
|
||||
if err := ensurePortAvailableOnIP(config.port, config.name, bindIp, reservedPorts, config.flagName); err != nil {
|
||||
if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, reservedPorts, config.flagName, config.localhost); err != nil {
|
||||
return err
|
||||
}
|
||||
// If port was changed, update the reserved gRPC ports mapping
|
||||
@@ -975,7 +1000,7 @@ func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
// Initialize all gRPC ports before services start
|
||||
// This ensures they won't be recalculated and cause conflicts
|
||||
// All gRPC port handling (calculation, validation, and assignment) is performed exclusively in initializeGrpcPortsOnIP
|
||||
allocatedPorts := initializeGrpcPortsOnIP(bindIp)
|
||||
allocatedPorts := initializeGrpcPortsOnIP(bindIp, listenOnLocalhost)
|
||||
|
||||
// Every other service binds within a moment of this check, but the admin
|
||||
// waits for all of them first. The admin gRPC port sits inside the Linux
|
||||
@@ -1054,7 +1079,7 @@ func listenMiniAdminWorker(bindIP string, port int) (net.Listener, error) {
|
||||
// This must be called after HTTP ports are finalized and before services start
|
||||
// Returns the set of ports allocated to mini services, for callers that must
|
||||
// keep optional listeners from stealing them.
|
||||
func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
func initializeGrpcPortsOnIP(bindIp string, listenOnLocalhost bool) map[int]bool {
|
||||
// Track all ports allocated (both HTTP and gRPC) to prevent collisions.
|
||||
// We must reserve HTTP ports so that gRPC fallback allocation never picks
|
||||
// a port already assigned to an HTTP service (which hasn't bound yet).
|
||||
@@ -1077,26 +1102,34 @@ func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
}
|
||||
|
||||
grpcConfigs := []struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{
|
||||
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master"},
|
||||
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer"},
|
||||
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume"},
|
||||
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master", bindIp, listenOnLocalhost},
|
||||
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer", bindIp, listenOnLocalhost},
|
||||
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume", bindIp, listenOnLocalhost},
|
||||
}
|
||||
if *miniEnableS3 {
|
||||
grpcConfigs = append(grpcConfigs, struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
}{miniS3Options.port, miniS3Options.portGrpc, "S3"})
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{miniS3Options.port, miniS3Options.portGrpc, "S3", bindIp, listenOnLocalhost})
|
||||
}
|
||||
// The Admin worker gRPC binds workerGrpcBindIp (loopback by default), not
|
||||
// -ip.bind, and never opens a companion listener.
|
||||
grpcConfigs = append(grpcConfigs, struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
}{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin"})
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin", miniAdminOptions.workerGrpcBindIp, false})
|
||||
|
||||
for _, config := range grpcConfigs {
|
||||
if config.grpcPort == nil {
|
||||
@@ -1110,10 +1143,10 @@ func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
|
||||
// Verify the gRPC port is available (whether calculated or explicitly set)
|
||||
// Check on both specific IP and all interfaces, and check against already allocated ports
|
||||
if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) || allocatedPorts[*config.grpcPort] {
|
||||
if !isPortUsableOnIP(config.ip, *config.grpcPort, config.localhost) || allocatedPorts[*config.grpcPort] {
|
||||
glog.Warningf("gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name)
|
||||
originalPort := *config.grpcPort
|
||||
newPort := findAvailablePortOnIP(bindIp, originalPort+1, 100, allocatedPorts)
|
||||
newPort := findAvailablePortOnIP(config.ip, originalPort+1, 100, allocatedPorts, config.localhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available gRPC port for %s starting from %d, will use %d and fail on binding", config.name, originalPort+1, originalPort)
|
||||
} else {
|
||||
@@ -1404,8 +1437,14 @@ func runMini(cmd *Command, args []string) bool {
|
||||
miniAdminOptions.workerGrpcBindIp = "127.0.0.1"
|
||||
}
|
||||
|
||||
// Services keep the loopback companion listener only while -ip.bind is
|
||||
// unset and the bind address falls back to -ip. While it is on, each
|
||||
// service port must be free on every interface, not just bindIp. This
|
||||
// must be captured before *miniBindIp is filled in below.
|
||||
listenOnLocalhost := *miniBindIp == "" && !util.IsLoopbackOrWildcardAddress(bindIp)
|
||||
|
||||
// Ensure all ports are available, find alternatives if needed
|
||||
if err := ensureAllPortsAvailableOnIP(bindIp); err != nil {
|
||||
if err := ensureAllPortsAvailableOnIP(bindIp, listenOnLocalhost); err != nil {
|
||||
glog.Errorf("Port allocation failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -1430,14 +1469,18 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// ip address
|
||||
miniMasterOptions.ip = miniIp
|
||||
miniMasterOptions.ipBind = miniBindIp
|
||||
miniMasterOptions.listenOnLocalhost = &listenOnLocalhost
|
||||
miniFilerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
|
||||
miniFilerOptions.ip = miniIp
|
||||
miniFilerOptions.bindIp = miniBindIp
|
||||
miniFilerOptions.listenOnLocalhost = &listenOnLocalhost
|
||||
miniS3Options.ip = miniIp
|
||||
miniS3Options.bindIp = miniBindIp
|
||||
miniS3Options.listenOnLocalhost = &listenOnLocalhost
|
||||
miniWebDavOptions.ipBind = miniBindIp
|
||||
miniOptions.v.ip = miniIp
|
||||
miniOptions.v.bindIp = miniBindIp
|
||||
miniOptions.v.listenOnLocalhost = &listenOnLocalhost
|
||||
miniOptions.v.masters = pb.ServerAddresses(actualPeersForComponents).ToAddresses()
|
||||
miniOptions.v.idleConnectionTimeout = miniTimeout
|
||||
// Admin server binds to the same address as the other mini services.
|
||||
@@ -1521,6 +1564,22 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// tear those services down too. On Ctrl+C they rely on their own
|
||||
// OnInterrupt hooks (see grace's LIFO ordering).
|
||||
miniMasterOptions.shutdownCtx = MiniClusterCtx
|
||||
// In single-master mode mini's embedded volume server is the only reporter
|
||||
// of prior cluster state; its first heartbeat already carries the full
|
||||
// inventory, so the post-mint allocation hold ends there instead of
|
||||
// running the whole warmup window on every fresh start. The master pins
|
||||
// the embedded address and refuses any other volume server, keeping the
|
||||
// single-reporter assumption true. Multi-master minis can field
|
||||
// late-reporting volume servers and keep the regular hold.
|
||||
// Both fields are re-derived on every run: the package-level options
|
||||
// survive in-process reruns, so a multi-master mini after a
|
||||
// single-master one must not inherit the release or the stale pin.
|
||||
singleMaster := isSingleMasterMode(*miniMasterOptions.peers)
|
||||
miniMasterOptions.releaseAllocOnFirstReport = singleMaster
|
||||
miniMasterOptions.singleReporterAddr = ""
|
||||
if singleMaster {
|
||||
miniMasterOptions.singleReporterAddr = util.JoinHostPort(*miniIp, *miniOptions.v.port)
|
||||
}
|
||||
miniOptions.v.shutdownCtx = MiniClusterCtx
|
||||
miniFilerOptions.shutdownCtx = MiniClusterCtx
|
||||
// Mini is a small/dev setup with short-lived RPCs; cap the filer's
|
||||
@@ -1776,6 +1835,14 @@ func miniAdminBindIP(requestedIP string, passwordConfigured, mtlsConfigured, all
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func miniAdminEffectiveBindIP(requestedIP string) string {
|
||||
hasMTLS := util.GetViper().GetString("https.admin.key") != "" &&
|
||||
util.GetViper().GetString("https.admin.ca") != ""
|
||||
allowInsecure := miniAdminOptions.allowInsecureBind != nil &&
|
||||
*miniAdminOptions.allowInsecureBind
|
||||
return miniAdminBindIP(requestedIP, *miniAdminOptions.adminPassword != "", hasMTLS, allowInsecure)
|
||||
}
|
||||
|
||||
// miniAdminWorkerAddress encodes the finalized HTTP and gRPC ports in the
|
||||
// server-address format understood by pb.ServerToGrpcAddress.
|
||||
func miniAdminWorkerAddress(bindIP string, httpPort, grpcPort int) string {
|
||||
@@ -1833,16 +1900,11 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
applyMiniAdminCredentialFallback(&miniAdminOptions)
|
||||
|
||||
requestedBindIP := getBindIp()
|
||||
bindIP := miniAdminEffectiveBindIP(requestedBindIP)
|
||||
hasMTLS := util.GetViper().GetString("https.admin.key") != "" &&
|
||||
util.GetViper().GetString("https.admin.ca") != ""
|
||||
allowInsecure := miniAdminOptions.allowInsecureBind != nil &&
|
||||
*miniAdminOptions.allowInsecureBind
|
||||
bindIP := miniAdminBindIP(
|
||||
requestedBindIP,
|
||||
*miniAdminOptions.adminPassword != "",
|
||||
hasMTLS,
|
||||
allowInsecure,
|
||||
)
|
||||
miniAdminOptions.ip = &bindIP
|
||||
if bindIP != requestedBindIP {
|
||||
glog.Warningf(
|
||||
|
||||
@@ -17,17 +17,20 @@ var (
|
||||
)
|
||||
|
||||
type MessageQueueAgentOptions struct {
|
||||
brokers []pb.ServerAddress
|
||||
brokersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
port *int
|
||||
brokers []pb.ServerAddress
|
||||
brokersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdMqAgent.Run = runMqAgent // break init cycle
|
||||
mqAgentOptions.brokersString = cmdMqAgent.Flag.String("broker", "localhost:17777", "comma-separated message queue brokers")
|
||||
mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "", "message queue agent host address")
|
||||
mqAgentOptions.ipBind = cmdMqAgent.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
mqAgentOptions.port = cmdMqAgent.Flag.Int("port", 16777, "message queue agent gRPC server port")
|
||||
}
|
||||
|
||||
@@ -60,8 +63,20 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
|
||||
SeedBrokers: mqAgentOpt.brokers,
|
||||
}, grpcDialOption)
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset and
|
||||
// the bind address falls back to -ip. An explicit -ip.bind pins the port to
|
||||
// exactly that address.
|
||||
bindIp := *mqAgentOpt.ip
|
||||
listenOnLocalhost := mqAgentOpt.ipBind == nil || *mqAgentOpt.ipBind == ""
|
||||
if mqAgentOpt.ipBind != nil && *mqAgentOpt.ipBind != "" {
|
||||
bindIp = *mqAgentOpt.ipBind
|
||||
}
|
||||
if mqAgentOpt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *mqAgentOpt.listenOnLocalhost
|
||||
}
|
||||
|
||||
// start grpc listener
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(*mqAgentOpt.ip, *mqAgentOpt.port, 0)
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(bindIp, *mqAgentOpt.port, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *mqAgentOpt.port, err)
|
||||
}
|
||||
@@ -84,7 +99,7 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
|
||||
}()
|
||||
}
|
||||
|
||||
glog.Infof("Start Seaweed Message Queue Agent on %s:%d", *mqAgentOpt.ip, *mqAgentOpt.port)
|
||||
glog.Infof("Start Seaweed Message Queue Agent on %s:%d", bindIp, *mqAgentOpt.port)
|
||||
if err := grpcS.Serve(grpcL); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("MQ Agent failed to start: %v", err)
|
||||
}
|
||||
|
||||
+24
-14
@@ -18,18 +18,20 @@ var (
|
||||
)
|
||||
|
||||
type MessageQueueBrokerOptions struct {
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
port *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
logFlushInterval *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
logFlushInterval *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -37,6 +39,7 @@ func init() {
|
||||
mqBrokerStandaloneOptions.mastersString = cmdMqBroker.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
|
||||
mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
|
||||
mqBrokerStandaloneOptions.ipBind = cmdMqBroker.Flag.String("ip.bind", "", "ip address to bind to. If empty, listen on all interfaces.")
|
||||
mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
|
||||
mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
|
||||
@@ -94,8 +97,15 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
|
||||
glog.Fatalf("failed to create new message broker for queue server: %v", err)
|
||||
}
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset.
|
||||
// An explicit -ip.bind pins the port to exactly that address.
|
||||
listenOnLocalhost := *mqBrokerOpt.ipBind == ""
|
||||
if mqBrokerOpt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *mqBrokerOpt.listenOnLocalhost
|
||||
}
|
||||
|
||||
// start grpc listener
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(*mqBrokerOpt.ipBind, *mqBrokerOpt.port, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
|
||||
}
|
||||
@@ -118,7 +128,7 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
|
||||
}()
|
||||
}
|
||||
|
||||
glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ip, *mqBrokerOpt.port)
|
||||
glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ipBind, *mqBrokerOpt.port)
|
||||
if err := grpcS.Serve(grpcL); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("Failed to serve MQ Broker: %v", err)
|
||||
}
|
||||
|
||||
+16
-9
@@ -46,6 +46,7 @@ type S3Options struct {
|
||||
filer *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
portHttps *int
|
||||
portGrpc *int
|
||||
@@ -359,6 +360,12 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
if *s3opt.portGrpc == 0 {
|
||||
*s3opt.portGrpc = 10000 + *s3opt.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset.
|
||||
// An explicit -ip.bind pins each port to exactly that address.
|
||||
listenOnLocalhost := *s3opt.bindIp == ""
|
||||
if s3opt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *s3opt.listenOnLocalhost
|
||||
}
|
||||
if *s3opt.bindIp == "" {
|
||||
*s3opt.bindIp = "0.0.0.0"
|
||||
}
|
||||
@@ -414,14 +421,14 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
// Start Iceberg REST Catalog server if enabled
|
||||
if *s3opt.portIceberg > 0 {
|
||||
if httpS := s3opt.startIcebergServer(s3ApiServer); httpS != nil {
|
||||
if httpS := s3opt.startIcebergServer(s3ApiServer, listenOnLocalhost); httpS != nil {
|
||||
httpServers = append(httpServers, httpS)
|
||||
}
|
||||
}
|
||||
|
||||
// Start Lance Namespace server if enabled
|
||||
if s3opt.portLance != nil && *s3opt.portLance > 0 {
|
||||
if httpS := s3opt.startLanceServer(s3ApiServer); httpS != nil {
|
||||
if httpS := s3opt.startLanceServer(s3ApiServer, listenOnLocalhost); httpS != nil {
|
||||
httpServers = append(httpServers, httpS)
|
||||
}
|
||||
}
|
||||
@@ -445,7 +452,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
|
||||
s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
@@ -459,7 +466,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *s3opt.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
@@ -532,7 +539,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
} else {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
|
||||
s3ApiListenerHttps, s3ApiLocalListenerHttps, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("S3 API HTTPS listener on %s:%d error: %v", *s3opt.bindIp, *s3opt.portHttps, err)
|
||||
}
|
||||
@@ -592,7 +599,7 @@ func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiSer
|
||||
}
|
||||
|
||||
// startIcebergServer starts the Iceberg REST Catalog server on a separate port.
|
||||
func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http.Server {
|
||||
func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer, listenOnLocalhost bool) *http.Server {
|
||||
icebergRouter := mux.NewRouter().SkipClean(true)
|
||||
// warehouse/parent query values may legally contain ';', which Go's
|
||||
// url.ParseQuery would otherwise drop
|
||||
@@ -614,7 +621,7 @@ func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portIceberg)
|
||||
icebergListener, icebergLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portIceberg, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portIceberg, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
// The catalog is auxiliary to the S3 gateway; a port conflict must not
|
||||
// take the whole server down.
|
||||
@@ -645,7 +652,7 @@ func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http
|
||||
// startLanceServer starts the Lance Namespace server on a separate port. It
|
||||
// shares the Iceberg catalog's credential role: one deployment vends table
|
||||
// credentials one way, whichever catalog the client speaks to.
|
||||
func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) *http.Server {
|
||||
func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenOnLocalhost bool) *http.Server {
|
||||
lanceRouter := mux.NewRouter().SkipClean(true)
|
||||
lanceRouter.Use(util_http.EscapeSemicolonsInQuery)
|
||||
|
||||
@@ -660,7 +667,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) *http.S
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portLance)
|
||||
lanceListener, lanceLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portLance, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portLance, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Errorf("Lance Namespace listener on %s error: %v", listenAddress, err)
|
||||
return nil
|
||||
|
||||
@@ -42,14 +42,14 @@ func TestAuxListenerBindFailureIsNonFatal(t *testing.T) {
|
||||
}
|
||||
s3a := &s3api.S3ApiServer{}
|
||||
|
||||
if s := s3opt.startIcebergServer(s3a); s != nil {
|
||||
if s := s3opt.startIcebergServer(s3a, false); s != nil {
|
||||
s.Close()
|
||||
t.Error("startIcebergServer on an occupied port should return nil")
|
||||
}
|
||||
if s3opt.icebergBound.Load() {
|
||||
t.Error("icebergBound must stay false when the listener failed to bind")
|
||||
}
|
||||
if s := s3opt.startLanceServer(s3a); s != nil {
|
||||
if s := s3opt.startLanceServer(s3a, false); s != nil {
|
||||
s.Close()
|
||||
t.Error("startLanceServer on an occupied port should return nil")
|
||||
}
|
||||
|
||||
@@ -69,6 +69,9 @@
|
||||
# volume-slot deadband for shard placement: nodes whose free volume slots differ by at
|
||||
# most this much tie and fewer live shards wins; applied times data shards (0 = strict)
|
||||
# free_slots_deadband = 0
|
||||
# shard copy concurrency. Leave unset for one in-flight copy per destination disk.
|
||||
# 0 sends shards sequentially. A positive N is a global cap, still one per disk.
|
||||
# max_shard_transfers = 4
|
||||
# max retry attempts for a failed erasure coding job
|
||||
# retry_limit = 1
|
||||
# seconds to wait between retry attempts
|
||||
|
||||
@@ -63,6 +63,11 @@ copy_other = 1 # create n x 1 = n actual volumes
|
||||
threshold = 0.9 # create threshold
|
||||
reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
|
||||
disable = false # disables volume growth if true
|
||||
# allocation_deferral = "15s" # hold the first volume id allocation after a fresh
|
||||
# topology id is minted, giving a rebuilt master's
|
||||
# veteran volume servers time to report. The hold
|
||||
# lifts early once heartbeats stay clean for two
|
||||
# pulses. "0s" disables it (e.g. test setups).
|
||||
|
||||
# configuration flags for replication
|
||||
[master.replication]
|
||||
|
||||
Loaded 100 of 168 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user