mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc97217591 | ||
|
|
7664bb3b7f | ||
|
|
4ae402de0d | ||
|
|
ef72d3976e | ||
|
|
153ab668c9 | ||
|
|
6dd6ad512c | ||
|
|
515380c349 | ||
|
|
9620e7ebfe | ||
|
|
53834117a6 | ||
|
|
88b91cccba | ||
|
|
fefac028ec | ||
|
|
0c9872f842 | ||
|
|
da1d7444b8 | ||
|
|
d15396fc4d | ||
|
|
39bdbc18db | ||
|
|
403d51631b | ||
|
|
52d91008ac |
No files matched your search
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
e2e:
|
||||
name: FUSE Mount
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
ec-integration-tests:
|
||||
name: EC Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -33,7 +33,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 50
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
jobs:
|
||||
metadata-subscribe-integration:
|
||||
name: Metadata Subscribe Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -27,7 +27,7 @@ permissions:
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
@@ -270,7 +270,7 @@ jobs:
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -37,9 +37,13 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
@@ -120,7 +124,7 @@ jobs:
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -146,9 +150,13 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -22,7 +22,7 @@ permissions:
|
||||
jobs:
|
||||
tests:
|
||||
name: S3 Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -25,7 +25,7 @@ defaults:
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -26,7 +26,7 @@ defaults:
|
||||
jobs:
|
||||
s3-versioning-tests:
|
||||
name: S3 Versioning Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
|
||||
s3-versioning-compatibility:
|
||||
name: S3 Versioning Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
|
||||
s3-cors-compatibility:
|
||||
name: S3 CORS Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
|
||||
s3-retention-tests:
|
||||
name: S3 Retention Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -266,7 +266,7 @@ jobs:
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -339,7 +339,7 @@ jobs:
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -398,7 +398,7 @@ jobs:
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -469,7 +469,7 @@ jobs:
|
||||
|
||||
s3-retention-worm:
|
||||
name: S3 Retention WORM Integration Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -519,7 +519,7 @@ jobs:
|
||||
|
||||
s3-versioning-stress:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
@@ -588,7 +588,7 @@ jobs:
|
||||
s3-tagging-tests:
|
||||
# CI job for S3 object tagging tests
|
||||
name: S3 Tagging Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -644,7 +644,7 @@ jobs:
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# Unit tests for IAM components
|
||||
iam-unit-tests:
|
||||
name: IAM Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
# S3 IAM integration tests with SeaweedFS services
|
||||
s3-iam-integration-tests:
|
||||
name: S3 IAM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
# Distributed IAM tests
|
||||
s3-iam-distributed-tests:
|
||||
name: S3 IAM Distributed Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -263,7 +263,7 @@ jobs:
|
||||
# Performance and stress tests
|
||||
s3-iam-performance-tests:
|
||||
name: S3 IAM Performance Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Dedicated job for Keycloak integration tests
|
||||
s3-keycloak-integration-tests:
|
||||
name: S3 Keycloak Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -21,7 +21,7 @@ permissions:
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
# Unit tests for policy components
|
||||
policy-unit-tests:
|
||||
name: S3 Policy Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
# S3 Policy Variables Integration Tests
|
||||
s3-policy-variables-tests:
|
||||
name: S3 Policy Variables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
# S3 Policy Enforcement Integration Tests
|
||||
s3-policy-enforcement-tests:
|
||||
name: S3 Policy Enforcement Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -317,7 +317,7 @@ jobs:
|
||||
# Trusted Proxy Detection Tests
|
||||
trusted-proxy-tests:
|
||||
name: Trusted Proxy Detection Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -405,7 +405,7 @@ jobs:
|
||||
# S3 Policy Shell Integration Tests
|
||||
s3-policy-shell-tests:
|
||||
name: S3 Policy Shell Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -24,7 +24,7 @@ permissions:
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
|
||||
@@ -46,7 +46,7 @@ permissions:
|
||||
jobs:
|
||||
snowflake-s3compat-tests:
|
||||
name: Snowflake S3Compat API tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
WORK_DIR: /tmp/seaweedfs-snowflake-tests
|
||||
|
||||
@@ -20,7 +20,7 @@ permissions:
|
||||
jobs:
|
||||
s3-spark-issue-repro-tests:
|
||||
name: S3 Spark Issue Reproduction Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -36,7 +36,7 @@ defaults:
|
||||
jobs:
|
||||
s3-sse-integration-tests:
|
||||
name: S3 SSE Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
s3-sse-compatibility:
|
||||
name: S3 SSE Compatibility Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
|
||||
s3-sse-metadata-persistence:
|
||||
name: S3 SSE Metadata Persistence Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -210,7 +210,7 @@ jobs:
|
||||
|
||||
s3-sse-copy-operations:
|
||||
name: S3 SSE Copy Operations Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -260,7 +260,7 @@ jobs:
|
||||
|
||||
s3-sse-multipart:
|
||||
name: S3 SSE Multipart Upload Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
|
||||
s3-sse-performance:
|
||||
name: S3 SSE Performance Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
# Only run performance tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
|
||||
@@ -361,7 +361,7 @@ jobs:
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -23,7 +23,7 @@ permissions:
|
||||
jobs:
|
||||
s3-tables-tests:
|
||||
name: S3 Tables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
|
||||
iceberg-catalog-tests:
|
||||
name: Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
|
||||
trino-iceberg-catalog-tests:
|
||||
name: Trino Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -227,7 +227,7 @@ jobs:
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -304,7 +304,7 @@ jobs:
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
|
||||
clickhouse-iceberg-catalog-tests:
|
||||
name: ClickHouse Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -470,7 +470,7 @@ jobs:
|
||||
|
||||
olake-iceberg-catalog-tests:
|
||||
name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -585,7 +585,7 @@ jobs:
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -657,7 +657,7 @@ jobs:
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -731,7 +731,7 @@ jobs:
|
||||
|
||||
risingwave-catalog-tests:
|
||||
name: RisingWave Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -806,7 +806,7 @@ jobs:
|
||||
|
||||
sts-integration-tests:
|
||||
name: STS Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -905,7 +905,7 @@ jobs:
|
||||
|
||||
lakekeeper-integration-tests:
|
||||
name: Lakekeeper Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -969,7 +969,7 @@ jobs:
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1046,7 +1046,7 @@ jobs:
|
||||
|
||||
lancedb-namespace-tests:
|
||||
name: LanceDB Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1120,7 +1120,7 @@ jobs:
|
||||
|
||||
table-lifecycle-tests:
|
||||
name: Table Lifecycle Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1189,7 +1189,7 @@ jobs:
|
||||
|
||||
duckdb-lance-tests:
|
||||
name: DuckDB Lance Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1268,7 +1268,7 @@ jobs:
|
||||
|
||||
spark-lance-namespace-tests:
|
||||
name: Spark Lance Namespace Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1346,7 +1346,7 @@ jobs:
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -1412,7 +1412,7 @@ jobs:
|
||||
|
||||
s3-tables-fmt-check:
|
||||
name: S3 Tables Format Check
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
@@ -1464,7 +1464,7 @@ jobs:
|
||||
|
||||
s3-tables-vet:
|
||||
name: S3 Tables Go Vet Check
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
basic-s3-tests:
|
||||
name: Basic S3 tests (KV store)
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -387,7 +387,7 @@ jobs:
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -558,7 +558,7 @@ jobs:
|
||||
|
||||
cors-tests:
|
||||
name: S3 CORS tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -684,7 +684,7 @@ jobs:
|
||||
|
||||
copy-tests:
|
||||
name: SeaweedFS Custom S3 Copy tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -787,7 +787,7 @@ jobs:
|
||||
|
||||
sql-store-tests:
|
||||
name: Basic S3 tests (SQL store)
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -29,7 +29,7 @@ permissions:
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -29,7 +29,7 @@ env:
|
||||
jobs:
|
||||
sftp-integration:
|
||||
name: SFTP Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -24,7 +24,7 @@ defaults:
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
|
||||
@@ -35,7 +35,7 @@ env:
|
||||
jobs:
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.50"
|
||||
appVersion: "4.51"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.50.0
|
||||
version: 4.51.0
|
||||
@@ -166,6 +166,10 @@ message Entry {
|
||||
repeated FileChunk chunks = 3;
|
||||
FuseAttributes attributes = 4;
|
||||
map<string, bytes> extended = 5;
|
||||
// filer-local commit timestamp of the version stored here: the meta log
|
||||
// event's ts_ns for the mutation that produced this entry, or 0 for
|
||||
// entries written before the field existed
|
||||
int64 commit_ts_ns = 6;
|
||||
bytes hard_link_id = 7;
|
||||
int32 hard_link_counter = 8; // only exists in hard link meta data
|
||||
bytes content = 9; // if not empty, the file content
|
||||
@@ -281,6 +285,7 @@ message WriteCondition {
|
||||
IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value
|
||||
IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future
|
||||
IF_CHUNKS_EQUAL = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)
|
||||
IF_ENTRY_EQUAL = 10; // fail unless the stored entry equals expected_entry
|
||||
}
|
||||
// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current
|
||||
// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it
|
||||
@@ -309,6 +314,7 @@ message WriteCondition {
|
||||
string gate_key = 7; // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_value
|
||||
string gate_value = 8; // gate value (e.g. retention mode COMPLIANCE for governance bypass)
|
||||
repeated string fids = 9; // chunk fid strings for IF_CHUNKS_EQUAL
|
||||
Entry expected_entry = 10; // whole-entry comparison for IF_ENTRY_EQUAL
|
||||
}
|
||||
repeated Clause clauses = 1; // all must hold (logical AND)
|
||||
}
|
||||
@@ -323,6 +329,7 @@ enum FilerError {
|
||||
EXISTING_IS_FILE = 4; // cannot overwrite file with directory
|
||||
ENTRY_ALREADY_EXISTS = 5; // O_EXCL and entry already exists
|
||||
PRECONDITION_FAILED = 6; // WriteCondition not satisfied
|
||||
READ_ONLY = 7; // path matches a read-only storage rule (e.g. bucket over quota)
|
||||
}
|
||||
|
||||
// ObjectMutation is one entry-level change applied by ObjectTransaction. All
|
||||
@@ -351,6 +358,7 @@ message ObjectMutation {
|
||||
bytes content = 11; // PATCH_EXTENDED: new Entry.content when set_content
|
||||
bool touch_mtime = 12; // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
bool remove_empty_parent = 13; // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
bool keep_remote_object = 14; // DELETE: leave the object on the mounted remote storage
|
||||
}
|
||||
|
||||
// Recompute re-derives a pointer entry (directory/name on the mutation) from the
|
||||
@@ -388,6 +396,8 @@ message ObjectTransactionRequest {
|
||||
string condition_key = 6; // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)
|
||||
string route_key = 7; // ring key identifying the owner filer; a non-owner forwards the whole transaction to it
|
||||
bool is_moved = 8; // set on a forwarded transaction so the receiver applies it locally instead of forwarding again
|
||||
string request_id = 9; // idempotency token: the owner records the outcome and replays it on a retried request instead of re-applying
|
||||
string condition_alt_key = 10; // if the condition's entry is absent, evaluate it against this entry instead (e.g. null slot, else latest version)
|
||||
}
|
||||
|
||||
message ObjectTransactionResponse {
|
||||
@@ -477,6 +487,7 @@ message UpdateEntryRequest {
|
||||
// the write, under the filer's per-path lock. The caller must route the
|
||||
// key's writes to this entry's owner filer for the check to be authoritative.
|
||||
WriteCondition condition = 6;
|
||||
bool is_moved = 7; // set on a forwarded request so the owner applies it locally
|
||||
}
|
||||
message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
@@ -514,6 +525,7 @@ message DeleteEntryRequest {
|
||||
bool is_from_other_cluster = 7;
|
||||
repeated int32 signatures = 8;
|
||||
int64 if_not_modified_after = 9;
|
||||
bool keep_remote_object = 10;
|
||||
}
|
||||
|
||||
message DeleteEntryResponse {
|
||||
@@ -573,6 +585,9 @@ message AssignVolumeResponse {
|
||||
// fsync is the storage rule's fsync decision for the assigned path, so the
|
||||
// client can carry it onto the volume server upload request.
|
||||
bool fsync = 11;
|
||||
// error_code classifies error for callers that must branch on it; error
|
||||
// keeps the human-readable text for clients that predate the field.
|
||||
FilerError error_code = 12;
|
||||
}
|
||||
|
||||
message LookupVolumeRequest {
|
||||
@@ -709,6 +724,10 @@ message TraverseBfsMetadataRequest {
|
||||
message TraverseBfsMetadataResponse {
|
||||
string directory = 1;
|
||||
Entry entry = 2;
|
||||
// The sender's per-path commit timestamp (see meta_path_ts.go): the newest
|
||||
// event time this filer committed for the path, 0 when unrecorded so
|
||||
// subscribers fall back to the entry's mtime.
|
||||
int64 entry_ts_ns = 3;
|
||||
}
|
||||
|
||||
message LogEntry {
|
||||
|
||||
Generated
+1
@@ -4601,6 +4601,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex",
|
||||
"http-body 1.0.1",
|
||||
"httpdate",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"image",
|
||||
|
||||
@@ -51,6 +51,7 @@ prost-types = "0.14"
|
||||
# HTTP server
|
||||
axum = { version = "0.8", features = ["multipart"] }
|
||||
http-body = "1"
|
||||
httpdate = "1"
|
||||
hyper = { version = "1", features = ["full"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "service", "server-auto", "http1", "http2"] }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
|
||||
@@ -1120,6 +1120,10 @@ async fn get_or_head_handler_inner(
|
||||
};
|
||||
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
// A Range whose If-Range no longer matches is ignored and the whole
|
||||
// content is sent, so a resumed download restarts instead of mixing
|
||||
// two versions.
|
||||
let range = range.filter(|_| if_range_matches(&headers, &etag, &last_modified_str));
|
||||
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
|
||||
// A request that ends here was answered by the meta read alone.
|
||||
if let Some(io_errors) = strategy.io_errors() {
|
||||
@@ -1167,14 +1171,20 @@ async fn get_or_head_handler_inner(
|
||||
return head_from_meta_response(&info, response_headers);
|
||||
}
|
||||
ReadStrategy::RangeFromSource(info) => {
|
||||
return range_from_source_response(
|
||||
&state,
|
||||
range.as_deref().unwrap_or_default(),
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await;
|
||||
return match range.as_deref() {
|
||||
Some(range) => {
|
||||
range_from_source_response(
|
||||
&state,
|
||||
range,
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// If-Range dropped the range; serve the whole content.
|
||||
None => stream_response(&state, info, response_headers, track_download),
|
||||
};
|
||||
}
|
||||
ReadStrategy::Buffered => {}
|
||||
}
|
||||
@@ -1557,6 +1567,31 @@ fn not_modified_response(
|
||||
None
|
||||
}
|
||||
|
||||
/// If-Range (RFC 9110 13.1.5): an entity-tag by strong comparison, or a date
|
||||
/// equal to Last-Modified. The response etag here is always a strong tag.
|
||||
fn if_range_matches(headers: &HeaderMap, etag: &str, last_modified_str: &Option<String>) -> bool {
|
||||
let Some(raw) = headers.get(header::IF_RANGE) else {
|
||||
return true;
|
||||
};
|
||||
let Ok(if_range) = raw.to_str() else {
|
||||
return false;
|
||||
};
|
||||
let if_range = if_range.trim();
|
||||
if if_range.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if if_range.starts_with('"') || if_range.starts_with("W/") {
|
||||
return if_range == etag;
|
||||
}
|
||||
let Ok(date) = httpdate::parse_http_date(if_range) else {
|
||||
return false;
|
||||
};
|
||||
last_modified_str
|
||||
.as_deref()
|
||||
.and_then(|lm| httpdate::parse_http_date(lm).ok())
|
||||
.is_some_and(|lm| date == lm)
|
||||
}
|
||||
|
||||
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
|
||||
fn etag_list_matches(header: &str, etag: &str) -> bool {
|
||||
if header.trim() == "*" {
|
||||
@@ -5767,6 +5802,53 @@ mod tests {
|
||||
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_if_range_matches() {
|
||||
let etag = "\"v2\"".to_string();
|
||||
let last_modified_str = Some("Sat, 24 Oct 2026 22:01:04 GMT".to_string());
|
||||
let cases: Vec<(Option<&str>, bool)> = vec![
|
||||
(None, true),
|
||||
(Some("\"v2\""), true),
|
||||
(Some("Sat, 24 Oct 2026 22:01:04 GMT"), true),
|
||||
(Some("Saturday, 24-Oct-26 22:01:04 GMT"), true),
|
||||
(Some("Sat Oct 24 22:01:04 2026"), true),
|
||||
(Some("\"v1\""), false),
|
||||
(Some("W/\"v2\""), false),
|
||||
(Some("Sat, 24 Oct 2026 22:01:03 GMT"), false),
|
||||
(Some("not a date"), false),
|
||||
];
|
||||
for (if_range, want) in cases {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(v) = if_range {
|
||||
headers.insert(header::IF_RANGE, v.parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
if_range_matches(&headers, &etag, &last_modified_str),
|
||||
want,
|
||||
"{if_range:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// A present value that is not valid UTF-8 fails the condition.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
header::HeaderValue::from_bytes(b"\"\xff\xfev2\"").unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &last_modified_str));
|
||||
|
||||
// No Last-Modified on the representation: a date validator fails,
|
||||
// an entity-tag still applies.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::IF_RANGE, "\"v2\"".parse().unwrap());
|
||||
assert!(if_range_matches(&headers, &etag, &None));
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
"Sat, 24 Oct 2026 22:01:04 GMT".parse().unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &None));
|
||||
}
|
||||
|
||||
/// A GET whose needle read is parked must not hold the store lock: a
|
||||
/// writer, here a real append to the same volume, must get through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
@@ -34,6 +34,7 @@ struct ListTablesResponse {
|
||||
pub struct NamespaceClient {
|
||||
base_url: String,
|
||||
http: reqwest::Client,
|
||||
api_key: Option<String>,
|
||||
}
|
||||
|
||||
/// A namespace call that has not answered by now is not going to. Without this
|
||||
@@ -41,12 +42,23 @@ pub struct NamespaceClient {
|
||||
/// slot open forever, and the sweep never finishes.
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const MAX_REDIRECT_HOPS: usize = 10;
|
||||
|
||||
impl NamespaceClient {
|
||||
pub fn new(base_url: impl Into<String>) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
// reqwest forwards custom headers on a follow, so only a same-origin
|
||||
// hop is allowed: anywhere else would carry x-api-key to a listener
|
||||
// the credential was never meant for.
|
||||
.redirect(reqwest::redirect::Policy::custom(|attempt| {
|
||||
if may_follow(attempt.previous(), attempt.url()) {
|
||||
attempt.follow()
|
||||
} else {
|
||||
attempt.stop()
|
||||
}
|
||||
}))
|
||||
.build()
|
||||
// The builder only fails on a bad TLS backend, which would break
|
||||
// every call anyway; a client with no timeouts is worse than a panic
|
||||
@@ -58,6 +70,32 @@ impl NamespaceClient {
|
||||
Self {
|
||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||
http,
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The catalog accepts an S3 credential as `x-api-key`; where the worker
|
||||
/// already holds one for storage it authenticates the namespace calls too.
|
||||
pub fn with_fallback(mut self, fallback: &crate::dataset::FallbackOptions) -> Self {
|
||||
if let (Some(access), Some(secret)) = (
|
||||
fallback.get("aws_access_key_id"),
|
||||
fallback.get("aws_secret_access_key"),
|
||||
) {
|
||||
if !protected_transport(&self.base_url) {
|
||||
tracing::warn!(
|
||||
"catalog url {} exposes the x-api-key credential in cleartext",
|
||||
self.base_url
|
||||
);
|
||||
}
|
||||
self.api_key = Some(format!("{access}:{secret}"));
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match &self.api_key {
|
||||
Some(key) => request.header("x-api-key", key),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +103,7 @@ impl NamespaceClient {
|
||||
pub async fn list_all_tables(&self) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/table", self.base_url);
|
||||
let response: ListTablesResponse = self
|
||||
.http
|
||||
.get(&url)
|
||||
.authed(self.http.get(&url))
|
||||
.send()
|
||||
.await
|
||||
.context("list tables")?
|
||||
@@ -80,8 +117,7 @@ impl NamespaceClient {
|
||||
pub async fn list_namespaces(&self, id: &[String]) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/namespace/{}/list", self.base_url, encode_id(id));
|
||||
let response: ListNamespacesResponse = self
|
||||
.http
|
||||
.get(&url)
|
||||
.authed(self.http.get(&url))
|
||||
.send()
|
||||
.await
|
||||
.context("list namespaces")?
|
||||
@@ -98,8 +134,7 @@ impl NamespaceClient {
|
||||
let url = format!("{}/v1/table/{}/describe", self.base_url, encode_id(id));
|
||||
let body = serde_json::json!({ "id": id, "vend_credentials": true });
|
||||
let description: TableDescription = self
|
||||
.http
|
||||
.post(&url)
|
||||
.authed(self.http.post(&url))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
@@ -111,6 +146,41 @@ impl NamespaceClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a hop bound a catalog that redirects to itself keeps the request
|
||||
/// alive until the call timeout, so the custom policy keeps the limit the
|
||||
/// default one had.
|
||||
fn may_follow(previous: &[reqwest::Url], next: &reqwest::Url) -> bool {
|
||||
previous.len() < MAX_REDIRECT_HOPS
|
||||
&& previous.last().is_some_and(|prev| same_origin(prev, next))
|
||||
}
|
||||
|
||||
/// A redirect stays within an origin when scheme, host, and port all match, so
|
||||
/// a catalog cannot bounce the api key onto a downgraded or foreign listener.
|
||||
fn same_origin(from: &reqwest::Url, to: &reqwest::Url) -> bool {
|
||||
from.scheme() == to.scheme()
|
||||
&& from.host_str() == to.host_str()
|
||||
&& from.port_or_known_default() == to.port_or_known_default()
|
||||
}
|
||||
|
||||
/// A catalog url protects the api key when the hop is TLS or never leaves the
|
||||
/// host; anything else sends it where the network can read it.
|
||||
fn protected_transport(base_url: &str) -> bool {
|
||||
let Ok(url) = reqwest::Url::parse(base_url) else {
|
||||
return false;
|
||||
};
|
||||
if url.scheme() == "https" {
|
||||
return true;
|
||||
}
|
||||
match url.host_str() {
|
||||
Some("localhost") => true,
|
||||
Some(host) => host
|
||||
.trim_matches(['[', ']'])
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|ip| ip.is_loopback()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_id(id: &[String]) -> String {
|
||||
if id.is_empty() {
|
||||
DELIMITER.to_string()
|
||||
@@ -127,3 +197,56 @@ pub fn parse_id(encoded: &str) -> Vec<String> {
|
||||
.map(|part| part.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin};
|
||||
|
||||
#[test]
|
||||
fn may_follow_bounds_hops_within_the_origin() {
|
||||
let url = |u: &str| reqwest::Url::parse(u).unwrap();
|
||||
let first = [url("https://c.local/ns")];
|
||||
let next = url("https://c.local/other");
|
||||
assert!(may_follow(&first, &next));
|
||||
assert!(!may_follow(&[], &next));
|
||||
assert!(!may_follow(&first, &url("http://c.local/other")));
|
||||
let full: Vec<_> = (0..MAX_REDIRECT_HOPS)
|
||||
.map(|i| url(&format!("https://c.local/{i}")))
|
||||
.collect();
|
||||
assert!(!may_follow(&full, &next));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_origin_rejects_scheme_and_port_hops() {
|
||||
for (from, to, expected) in [
|
||||
("https://c.local/ns", "https://c.local/other", true),
|
||||
("http://c.local", "http://c.local:80/next", true),
|
||||
("https://c.local", "http://c.local/next", false),
|
||||
("https://c.local:8443", "https://c.local/next", false),
|
||||
("https://c.local", "https://other.local/next", false),
|
||||
] {
|
||||
let from = reqwest::Url::parse(from).unwrap();
|
||||
let to = reqwest::Url::parse(to).unwrap();
|
||||
assert_eq!(same_origin(&from, &to), expected, "{from} -> {to}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_transport_covers_tls_and_loopback() {
|
||||
for url in [
|
||||
"https://catalog.internal:8443",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://localhost",
|
||||
"http://[::1]:8080",
|
||||
] {
|
||||
assert!(protected_transport(url), "{url} should be protected");
|
||||
}
|
||||
for url in [
|
||||
"http://catalog.internal",
|
||||
"http://127.0.0.1.evil.test",
|
||||
"not a url",
|
||||
] {
|
||||
assert!(!protected_transport(url), "{url} should warn");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ impl CleanupVersionsHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ impl CompactHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ impl OptimizeIndicesHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ impl SortHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ impl PreviewProvider for LancePreview {
|
||||
}
|
||||
|
||||
async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
|
||||
let client = NamespaceClient::new(self.namespace_url.clone());
|
||||
let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback);
|
||||
let table = dataset::open(&client, object_id, &self.fallback).await?;
|
||||
|
||||
let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
|
||||
|
||||
@@ -244,7 +244,13 @@ func (env *environment) tableBucketExists(t *testing.T, bucket string) bool {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s:%d/v1/namespace/%s/exists", env.bindIP, env.lancePort, bucket)
|
||||
resp, err := http.Post(url, "application/json", strings.NewReader("{}"))
|
||||
req, err := http.NewRequest(http.MethodPost, url, strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("build the namespace exists request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", env.accessKey+":"+env.secretKey)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("ask the namespace whether %s exists: %v", bucket, err)
|
||||
}
|
||||
|
||||
@@ -53,7 +53,13 @@ def main():
|
||||
"aws_region": "us-east-1",
|
||||
}
|
||||
|
||||
ns = ln.connect("rest", {"uri": args.namespace_url})
|
||||
ns = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
ns.create_namespace(ln.CreateNamespaceRequest(id=[args.bucket], mode="EXIST_OK"))
|
||||
ns.create_namespace(
|
||||
ln.CreateNamespaceRequest(id=[args.bucket, args.namespace], mode="EXIST_OK")
|
||||
|
||||
@@ -45,7 +45,15 @@ def seed_table(namespace_url, storage, bucket, namespace, table, rows):
|
||||
LanceDB reads through the catalog; the writing half is pylance, because the
|
||||
namespace records where a table lives and does not carry its data.
|
||||
"""
|
||||
ns = ln.connect("rest", {"uri": namespace_url})
|
||||
ns = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": namespace_url,
|
||||
"header.x-api-key": (
|
||||
f"{storage['aws_access_key_id']}:{storage['aws_secret_access_key']}"
|
||||
),
|
||||
},
|
||||
)
|
||||
ns.create_namespace(
|
||||
ln.CreateNamespaceRequest(id=[bucket], mode="EXIST_OK")
|
||||
)
|
||||
@@ -98,7 +106,12 @@ def main():
|
||||
|
||||
print(f"lancedb {lancedb.__version__} connecting to {args.namespace_url}")
|
||||
db = lancedb.connect_namespace(
|
||||
"rest", {"uri": args.namespace_url}, storage_options=storage
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
storage_options=storage,
|
||||
)
|
||||
|
||||
# 1. The catalog is browsable: the bucket is a namespace, and the table is
|
||||
@@ -190,7 +203,10 @@ def main():
|
||||
# declare-and-write - rather than a hang, a 404, or a half-made table.
|
||||
pushdown = lancedb.connect_namespace(
|
||||
"rest",
|
||||
{"uri": args.namespace_url},
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
storage_options=storage,
|
||||
namespace_client_pushdown_operations=["CreateTable"],
|
||||
)
|
||||
|
||||
@@ -41,6 +41,7 @@ deployment uses whatever address the gateway answers on:
|
||||
spark.sql.catalog.lance org.lance.spark.LanceNamespaceSparkCatalog
|
||||
spark.sql.catalog.lance.impl rest
|
||||
spark.sql.catalog.lance.uri http://seaweed:9101
|
||||
spark.sql.catalog.lance.headers.x-api-key <access-key>:<secret-key>
|
||||
spark.sql.catalog.lance.storage.aws_endpoint http://seaweed:8333
|
||||
spark.sql.catalog.lance.storage.allow_http true
|
||||
spark.sql.catalog.lance.storage.aws_access_key_id …
|
||||
|
||||
@@ -46,6 +46,10 @@ def main():
|
||||
.config("spark.sql.catalog.lance", "org.lance.spark.LanceNamespaceSparkCatalog")
|
||||
.config("spark.sql.catalog.lance.impl", "rest")
|
||||
.config("spark.sql.catalog.lance.uri", args.namespace_url)
|
||||
.config(
|
||||
"spark.sql.catalog.lance.headers.x-api-key",
|
||||
f"{args.access_key}:{args.secret_key}",
|
||||
)
|
||||
.config("spark.sql.catalog.lance.storage.aws_endpoint", args.s3_endpoint)
|
||||
.config("spark.sql.catalog.lance.storage.allow_http", "true")
|
||||
.config("spark.sql.catalog.lance.storage.aws_access_key_id", args.access_key)
|
||||
|
||||
@@ -239,7 +239,13 @@ func (env *environment) tableBucketExists(t *testing.T, bucket string) bool {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s:%d/v1/namespace/%s/exists", env.bindIP, env.lancePort, bucket)
|
||||
resp, err := http.Post(url, "application/json", strings.NewReader("{}"))
|
||||
req, err := http.NewRequest(http.MethodPost, url, strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("build the namespace exists request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", env.accessKey+":"+env.secretKey)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("ask the namespace whether %s exists: %v", bucket, err)
|
||||
}
|
||||
|
||||
@@ -99,7 +99,8 @@ def connect(args):
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": f"s3://{args.bucket}/",
|
||||
"prefix": args.bucket,
|
||||
"auth": {"type": "noop"},
|
||||
"oauth2-server-uri": f"{args.catalog_url}/v1/oauth/tokens",
|
||||
"credential": f"{args.access_key}:{args.secret_key}",
|
||||
"s3.endpoint": args.s3_endpoint,
|
||||
"s3.access-key-id": args.access_key,
|
||||
"s3.secret-access-key": args.secret_key,
|
||||
|
||||
@@ -83,7 +83,13 @@ def tally(dataset):
|
||||
|
||||
def resolve(args):
|
||||
"""Ask the namespace where the table lives, the way the worker does."""
|
||||
namespace = ln.connect("rest", {"uri": args.namespace_url})
|
||||
namespace = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
table_id = [args.bucket, args.namespace, args.table]
|
||||
described = namespace.describe_table(ln.DescribeTableRequest(id=table_id))
|
||||
return namespace, table_id, described.location, storage_options(args, described)
|
||||
@@ -115,7 +121,13 @@ def main():
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.phase == "write":
|
||||
namespace = ln.connect("rest", {"uri": args.namespace_url})
|
||||
namespace = ln.connect(
|
||||
"rest",
|
||||
{
|
||||
"uri": args.namespace_url,
|
||||
"header.x-api-key": f"{args.access_key}:{args.secret_key}",
|
||||
},
|
||||
)
|
||||
for parent in ([args.bucket], [args.bucket, args.namespace]):
|
||||
namespace.create_namespace(ln.CreateNamespaceRequest(id=parent, mode="EXIST_OK"))
|
||||
table_id = [args.bucket, args.namespace, args.table]
|
||||
|
||||
@@ -111,6 +111,16 @@ func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error
|
||||
}
|
||||
ecChanged = true
|
||||
}
|
||||
if k := "maintenance.erasure_coding.max_shard_transfers"; v.IsSet(k) {
|
||||
n := v.GetInt(k)
|
||||
if n < 0 {
|
||||
n = 0
|
||||
} else if n > math.MaxInt32 {
|
||||
n = math.MaxInt32
|
||||
}
|
||||
ecConf.MaxShardTransfers = &n
|
||||
ecChanged = true
|
||||
}
|
||||
|
||||
if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged {
|
||||
return nil
|
||||
@@ -239,6 +249,7 @@ var pluginConfigSections = []pluginConfigSection{
|
||||
"replica_placement": stringValue,
|
||||
"strict_placement": boolValue,
|
||||
"free_slots_deadband": int64Value,
|
||||
"max_shard_transfers": int64Value,
|
||||
},
|
||||
// workers read collection_filter from the admin values, not the worker values
|
||||
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
|
||||
|
||||
@@ -307,3 +307,97 @@ preferred_tags = ["fast,SSD"]
|
||||
t.Errorf("preferred_tags = %v, want [fast ssd]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMaintenanceConfigFromTomlMaxShardTransfers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cp := NewConfigPersistence(dir)
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
fullness_ratio = 0.9
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply fullness: %v", err)
|
||||
}
|
||||
ec := erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers != nil {
|
||||
t.Fatalf("absent key persisted as %d, want unset", *ec.MaxShardTransfers)
|
||||
}
|
||||
if ec.FullnessRatio != 0.9 {
|
||||
t.Fatalf("fullness_ratio = %v, want 0.9", ec.FullnessRatio)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 0
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 0: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 0 {
|
||||
t.Fatalf("0: got %v, want sequential 0", ec.MaxShardTransfers)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 4
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 4: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 4 {
|
||||
t.Fatalf("4: got %v, want 4", ec.MaxShardTransfers)
|
||||
}
|
||||
|
||||
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = -3
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply negative: %v", err)
|
||||
}
|
||||
ec = erasure_coding.LoadConfigFromPersistence(cp)
|
||||
if ec.MaxShardTransfers == nil || *ec.MaxShardTransfers != 0 {
|
||||
t.Fatalf("negative: got %v, want sequential 0", ec.MaxShardTransfers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPluginConfigFromTomlMaxShardTransfers(t *testing.T) {
|
||||
s := newPluginServer(t)
|
||||
seed := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "erasure_coding",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
|
||||
}
|
||||
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
if err := s.ApplyPluginConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
replica_placement = "020"
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply placement: %v", err)
|
||||
}
|
||||
cfg, err := s.GetPlugin().LoadJobTypeConfig("erasure_coding")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load: %v %v", cfg, err)
|
||||
}
|
||||
if _, ok := cfg.WorkerConfigValues["max_shard_transfers"]; ok {
|
||||
t.Fatal("absent max_shard_transfers was written into worker config")
|
||||
}
|
||||
|
||||
if err := s.ApplyPluginConfigFromToml(tomlConfig(t, `
|
||||
[maintenance.erasure_coding]
|
||||
max_shard_transfers = 0
|
||||
`)); err != nil {
|
||||
t.Fatalf("apply 0: %v", err)
|
||||
}
|
||||
cfg, err = s.GetPlugin().LoadJobTypeConfig("erasure_coding")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load 0: %v %v", cfg, err)
|
||||
}
|
||||
got, ok := cfg.WorkerConfigValues["max_shard_transfers"]
|
||||
if !ok {
|
||||
t.Fatal("max_shard_transfers = 0 missing from worker config")
|
||||
}
|
||||
if got.GetInt64Value() != 0 {
|
||||
t.Fatalf("max_shard_transfers = %d, want 0", got.GetInt64Value())
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -50,9 +51,14 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers
|
||||
// ShowFileBrowser renders the file browser page
|
||||
func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
|
||||
// Get path from query parameter, default to root
|
||||
path := defaultQuery(r.URL.Query().Get("path"), "/")
|
||||
// Normalize Windows-style paths for consistency
|
||||
path = util.CleanWindowsPath(path)
|
||||
dirPath := path.Clean(util.CleanWindowsPath(defaultQuery(r.URL.Query().Get("path"), "/")))
|
||||
if !strings.HasPrefix(dirPath, "/") {
|
||||
dirPath = "/" + dirPath
|
||||
}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) && restrictedBrowserPath(dirPath) {
|
||||
http.Redirect(w, r, dash.P(r.Context(), "/admin?error=Insufficient permissions"), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
|
||||
// Get pagination parameters
|
||||
lastFileName := r.URL.Query().Get("lastFileName")
|
||||
@@ -66,7 +72,7 @@ func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// Get file browser data with cursor-based pagination
|
||||
browserData, err := h.adminServer.GetFileBrowser(path, "", lastFileName, pageSize)
|
||||
browserData, err := h.adminServer.GetFileBrowser(dirPath, "", lastFileName, pageSize)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get file browser data: "+err.Error())
|
||||
return
|
||||
@@ -356,14 +362,46 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
|
||||
cleanPath = "/" + cleanPath
|
||||
}
|
||||
|
||||
// Prevent path traversal attacks
|
||||
if strings.Contains(cleanPath, "..") {
|
||||
// Reject parent-directory components; names merely containing ".." are valid
|
||||
if slices.Contains(strings.Split(cleanPath, "/"), "..") {
|
||||
return "", fmt.Errorf("path traversal not allowed")
|
||||
}
|
||||
|
||||
return cleanPath, nil
|
||||
}
|
||||
|
||||
// restrictedBrowserPath reports whether a cleaned path lies inside the filer's
|
||||
// /etc tree, which holds the credential store and cluster configuration.
|
||||
func restrictedBrowserPath(cleanPath string) bool {
|
||||
return cleanPath == strings.TrimSuffix(filer.DirectoryEtcRoot, "/") ||
|
||||
strings.HasPrefix(cleanPath, filer.DirectoryEtcRoot)
|
||||
}
|
||||
|
||||
// cleanReadPath validates the request's path parameter; read-only sessions are
|
||||
// denied the /etc tree so the file browser cannot serve stored credentials.
|
||||
func (h *FileBrowserHandlers) cleanReadPath(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
cleanPath, err := h.validateAndCleanFilePath(r.URL.Query().Get("path"))
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return "", false
|
||||
}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) && restrictedBrowserPath(cleanPath) {
|
||||
writeJSONError(w, http.StatusForbidden, "Access to this path requires admin access")
|
||||
return "", false
|
||||
}
|
||||
return cleanPath, true
|
||||
}
|
||||
|
||||
// exportExcludedPrefixes lists the subtrees a metadata export skips; read-only
|
||||
// sessions also lose /etc so stored credentials stay out of the dump.
|
||||
func exportExcludedPrefixes(r *http.Request) []string {
|
||||
prefixes := []string{filer.SystemLogDir}
|
||||
if dash.IsReadOnlyRole(dash.RoleFromContext(r.Context())) {
|
||||
prefixes = append(prefixes, strings.TrimSuffix(filer.DirectoryEtcRoot, "/"))
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
|
||||
// fetchFileContent fetches file content via the filer gRPC service. It is
|
||||
// used for the "view as text" path, so the maxBytes cap matches the 1 MB
|
||||
// limit the caller already applies before invoking us.
|
||||
@@ -377,9 +415,8 @@ func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Dur
|
||||
// gRPC service, so the admin file browser keeps working even when the filer
|
||||
// is started with -disableHttp=true.
|
||||
func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
inline := r.URL.Query().Get("inline") == "true"
|
||||
@@ -419,9 +456,8 @@ func (t *responseWriteTracker) Write(p []byte) (int, error) {
|
||||
|
||||
// ViewFile handles file viewing requests (for text files, images, etc.)
|
||||
func (h *FileBrowserHandlers) ViewFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -524,9 +560,8 @@ func (h *FileBrowserHandlers) ViewFile(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// GetFileProperties handles file properties requests
|
||||
func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
filePath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -630,23 +665,18 @@ func (h *FileBrowserHandlers) GetFileProperties(w http.ResponseWriter, r *http.R
|
||||
// FullEntry stream — the format weed shell fs.meta.load reads. Directories are
|
||||
// walked recursively via the filer BFS metadata stream.
|
||||
func (h *FileBrowserHandlers) ExportMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
cleanPath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
cleanPath, ok := h.cleanReadPath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
excludedPrefixes := exportExcludedPrefixes(r)
|
||||
|
||||
tracker := &responseWriteTracker{ResponseWriter: w}
|
||||
|
||||
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
stream, err := client.TraverseBfsMetadata(r.Context(), &filer_pb.TraverseBfsMetadataRequest{
|
||||
Directory: cleanPath,
|
||||
ExcludedPrefixes: []string{filer.SystemLogDir},
|
||||
ExcludedPrefixes: excludedPrefixes,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -3,9 +3,12 @@ package handlers
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
func TestValidateAndCleanFilePath_AllowsControlChars(t *testing.T) {
|
||||
@@ -26,6 +29,9 @@ func TestValidateAndCleanFilePath_AllowsControlChars(t *testing.T) {
|
||||
{"relative/path.txt", "/relative/path.txt"},
|
||||
// Duplicate slashes should be collapsed by path.Clean.
|
||||
{"/a//b", "/a/b"},
|
||||
// ".." inside a file or directory name is not a parent component.
|
||||
{"/buckets/reports/report..csv", "/buckets/reports/report..csv"},
|
||||
{"/a../b", "/a../b"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := h.validateAndCleanFilePath(tc.in)
|
||||
@@ -46,6 +52,81 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAndCleanFilePath_RejectsParentComponents(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
for _, p := range []string{"../etc", "../../etc/iam", "..", "a/../../b"} {
|
||||
if _, err := h.validateAndCleanFilePath(p); err == nil {
|
||||
t.Errorf("validateAndCleanFilePath(%q): expected traversal rejection", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_DeniesEtcForReadOnly(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
denied := []string{"/etc", "/etc/iam", "/etc/iam/identities/root.json",
|
||||
"/etc/s3/kek.json", "/etc/../etc/iam/identity.json", "/a/../../etc/x"}
|
||||
for _, p := range denied {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path="+p, nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); ok {
|
||||
t.Errorf("path %q: expected denial for read-only session", p)
|
||||
continue
|
||||
}
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Errorf("path %q: expected 403, got %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_AllowsEtcForAdminAndNorRole(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
for _, role := range []string{"admin", ""} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path=/etc/iam/identities/root.json", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "u", role, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); !ok {
|
||||
t.Errorf("role %q: expected /etc path to be allowed, got %d", role, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanReadPath_AllowsReadOnlyOutsideEtc(t *testing.T) {
|
||||
h := &FileBrowserHandlers{}
|
||||
|
||||
for _, p := range []string{"/buckets/foo/bar.txt", "/etcx/file", "/etcetera", "/"} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/download?path="+p, nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if _, ok := h.cleanReadPath(w, req); !ok {
|
||||
t.Errorf("path %q: expected allowed for read-only session, got %d", p, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportExcludedPrefixes_ReadOnlyDropsEtc(t *testing.T) {
|
||||
etcPrefix := strings.TrimSuffix(filer.DirectoryEtcRoot, "/")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/metadata/export?path=/", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "ro", dash.RoleReadOnly, "csrf"))
|
||||
if got := exportExcludedPrefixes(req); !slices.Contains(got, etcPrefix) {
|
||||
t.Errorf("read-only export should exclude %q, got %v", etcPrefix, got)
|
||||
}
|
||||
|
||||
for _, role := range []string{"admin", ""} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/files/metadata/export?path=/", nil)
|
||||
req = req.WithContext(dash.WithAuthContext(req.Context(), "u", role, "csrf"))
|
||||
if got := exportExcludedPrefixes(req); slices.Contains(got, etcPrefix) {
|
||||
t.Errorf("role %q: export should keep %q, got %v", role, etcPrefix, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
|
||||
h := &FileBrowserHandlers{adminServer: &dash.AdminServer{}}
|
||||
|
||||
@@ -62,4 +143,3 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2272,6 +2272,105 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
return input;
|
||||
}
|
||||
|
||||
function shardTransferParallelOn(nativeValue) {
|
||||
if (nativeValue === null || nativeValue === undefined || nativeValue === '') {
|
||||
return true;
|
||||
}
|
||||
var n = parseInt(nativeValue, 10);
|
||||
if (Number.isNaN(n)) {
|
||||
return true;
|
||||
}
|
||||
return n !== 0;
|
||||
}
|
||||
|
||||
function shardTransferMaxText(nativeValue) {
|
||||
if (nativeValue === null || nativeValue === undefined || nativeValue === '') {
|
||||
return '';
|
||||
}
|
||||
var n = parseInt(nativeValue, 10);
|
||||
if (Number.isNaN(n) || n <= 0) {
|
||||
return '';
|
||||
}
|
||||
return String(n);
|
||||
}
|
||||
|
||||
// Presentation of the single persisted max_shard_transfers value.
|
||||
// Unchecked stores 0. Checked with an empty max omits the key.
|
||||
function renderShardTransferControls(scope, field, nativeValue) {
|
||||
var parallelOn = shardTransferParallelOn(nativeValue);
|
||||
var wrapper = document.createElement('div');
|
||||
wrapper.className = 'mb-3 plugin-field';
|
||||
wrapper.dataset.scope = scope;
|
||||
wrapper.dataset.fieldName = field.name;
|
||||
wrapper.dataset.fieldType = normalizeEnum(field.field_type);
|
||||
wrapper.dataset.widget = 'SHARD_TRANSFERS';
|
||||
|
||||
var checkWrap = document.createElement('div');
|
||||
checkWrap.className = 'form-check form-switch mb-2';
|
||||
var box = document.createElement('input');
|
||||
box.type = 'checkbox';
|
||||
box.className = 'form-check-input';
|
||||
box.id = toInputID(scope, field.name) + '-parallel';
|
||||
box.checked = parallelOn;
|
||||
var boxLabel = document.createElement('label');
|
||||
boxLabel.className = 'form-check-label';
|
||||
boxLabel.setAttribute('for', box.id);
|
||||
boxLabel.textContent = 'Parallel shard distribution';
|
||||
checkWrap.appendChild(box);
|
||||
checkWrap.appendChild(boxLabel);
|
||||
wrapper.appendChild(checkWrap);
|
||||
|
||||
var numberLabel = document.createElement('label');
|
||||
numberLabel.className = 'form-label';
|
||||
numberLabel.setAttribute('for', toInputID(scope, field.name));
|
||||
numberLabel.textContent = 'Max transfers';
|
||||
wrapper.appendChild(numberLabel);
|
||||
|
||||
var input = document.createElement('input');
|
||||
input.type = 'number';
|
||||
input.min = '1';
|
||||
input.step = '1';
|
||||
input.className = 'form-control';
|
||||
input.id = toInputID(scope, field.name);
|
||||
input.placeholder = 'no global cap';
|
||||
input.value = shardTransferMaxText(nativeValue);
|
||||
input.disabled = !parallelOn;
|
||||
wrapper.appendChild(input);
|
||||
|
||||
box.addEventListener('change', function() {
|
||||
input.disabled = !box.checked;
|
||||
});
|
||||
|
||||
var help = document.createElement('div');
|
||||
help.className = 'form-text';
|
||||
help.textContent = field.description || 'Unchecked sends shards sequentially. Checked with an empty max uses one in-flight copy per destination disk. A number is a global cap, still one copy per disk.';
|
||||
wrapper.appendChild(help);
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
function collectShardTransferValue(scope) {
|
||||
var box = document.getElementById(toInputID(scope, 'max_shard_transfers') + '-parallel');
|
||||
var input = document.getElementById(toInputID(scope, 'max_shard_transfers'));
|
||||
if (!box || !input) {
|
||||
return null;
|
||||
}
|
||||
if (!box.checked) {
|
||||
return { int64_value: '0' };
|
||||
}
|
||||
var raw = String(input.value || '').trim();
|
||||
if (raw === '') {
|
||||
return null;
|
||||
}
|
||||
if (!/^\d+$/.test(raw)) {
|
||||
throw new Error('Invalid int64 value for field Max transfers');
|
||||
}
|
||||
var n = parseInt(raw, 10);
|
||||
if (n <= 0) {
|
||||
return { int64_value: '0' };
|
||||
}
|
||||
return { int64_value: String(n) };
|
||||
}
|
||||
|
||||
function renderConfigForm(scope, rootID, form) {
|
||||
var root = document.getElementById(rootID);
|
||||
if (!root) {
|
||||
@@ -2321,6 +2420,11 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
var configValue = resolveFieldConfigValue(scope, field);
|
||||
var nativeValue = configValueToNative(configValue);
|
||||
|
||||
if (field.name === 'max_shard_transfers') {
|
||||
sectionBody.appendChild(renderShardTransferControls(scope, field, nativeValue));
|
||||
continue;
|
||||
}
|
||||
|
||||
var wrapper = document.createElement('div');
|
||||
wrapper.className = 'mb-3 plugin-field';
|
||||
wrapper.dataset.scope = scope;
|
||||
@@ -2440,6 +2544,14 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.name === 'max_shard_transfers') {
|
||||
var shardCap = collectShardTransferValue(scope);
|
||||
if (shardCap !== null) {
|
||||
out[field.name] = shardCap;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var wrapper = document.querySelector('.plugin-field[data-scope="' + scope + '"][data-field-name="' + field.name + '"]');
|
||||
if (wrapper && wrapper.classList.contains('plugin-field-hidden')) {
|
||||
continue;
|
||||
|
||||
@@ -104,7 +104,7 @@ func Plugin(page string, initialJob string, lane string) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"bi bi-arrow-repeat me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"bi bi-graph-up-arrow me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"bi bi-sliders me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"bi bi-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"bi bi-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"bi bi-list-check me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-3 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-secondary h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Next Run</div><h4 class=\"mb-0 plugin-scheduler-next-run\">-</h4><div class=\"text-muted small plugin-scheduler-next-run-meta\"></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-bar-chart me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"bi bi-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Sequential scheduler with per-job runtime limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Max Runtime</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th><th>Last Run</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-people me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"bi bi-sliders me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-body\"><div class=Line truncated
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"bi bi-arrow-repeat me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"bi bi-graph-up-arrow me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"bi bi-sliders me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"bi bi-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"bi bi-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"bi bi-list-check me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-3 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div><div class=\"col-md-3 mb-3\"><div class=\"card border-secondary h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Next Run</div><h4 class=\"mb-0 plugin-scheduler-next-run\">-</h4><div class=\"text-muted small plugin-scheduler-next-run-meta\"></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-bar-chart me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"bi bi-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Sequential scheduler with per-job runtime limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Max Runtime</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th><th>Last Run</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"bi bi-people me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"bi bi-sliders me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-body\"><div class=Line truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
+18
-3
@@ -64,6 +64,7 @@ type FilerOptions struct {
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
portGrpc *int
|
||||
publicPort *int
|
||||
@@ -282,12 +283,17 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
}
|
||||
go stats_collect.StartMetricsServer(*f.metricsHttpIp, *f.metricsHttpPort)
|
||||
|
||||
// Embedded services keep their loopback companion listener only while
|
||||
// -ip.bind is unset and the bind address falls back to -ip. This must be
|
||||
// captured before startFiler fills *f.bindIp in place.
|
||||
filerBindIpUnset := *f.bindIp == ""
|
||||
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
filerS3Options.ip = f.ip
|
||||
if *filerS3Options.bindIp == "" {
|
||||
filerS3Options.listenOnLocalhost = &filerBindIpUnset
|
||||
filerS3Options.bindIp = f.bindIp
|
||||
}
|
||||
filerS3Options.localFilerSocket = f.localSocket
|
||||
@@ -320,6 +326,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
if *filerStartIam {
|
||||
filerIamOptions.filer = &filerAddress
|
||||
filerIamOptions.masters = f.mastersString
|
||||
filerIamOptions.listenOnLocalhost = &filerBindIpUnset
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerIamOptions.startIamServer()
|
||||
@@ -330,6 +337,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
if *filerStartSftp {
|
||||
filerSftpOptions.filer = &filerAddress
|
||||
if *filerSftpOptions.bindIp == "" {
|
||||
filerSftpOptions.listenOnLocalhost = &filerBindIpUnset
|
||||
filerSftpOptions.bindIp = f.bindIp
|
||||
}
|
||||
go func(delay time.Duration) {
|
||||
@@ -356,6 +364,13 @@ func (fo *FilerOptions) startFiler() {
|
||||
if *fo.portGrpc == 0 {
|
||||
*fo.portGrpc = 10000 + *fo.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset
|
||||
// and the bind address falls back to -ip. An explicit -ip.bind pins each
|
||||
// port to exactly that address.
|
||||
listenOnLocalhost := *fo.bindIp == ""
|
||||
if fo.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *fo.listenOnLocalhost
|
||||
}
|
||||
if *fo.bindIp == "" {
|
||||
*fo.bindIp = *fo.ip
|
||||
}
|
||||
@@ -433,7 +448,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
if *fo.publicPort != 0 {
|
||||
publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
|
||||
glog.V(0).Infoln("Start Seaweed filer server", version.Version(), "public at", publicListeningAddress)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
|
||||
}
|
||||
@@ -459,7 +474,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", version.Version(), *fo.ip, *fo.port)
|
||||
filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
|
||||
*fo.bindIp, *fo.port,
|
||||
time.Duration(10)*time.Second,
|
||||
time.Duration(10)*time.Second, listenOnLocalhost,
|
||||
)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer listener error: %v", e)
|
||||
@@ -467,7 +482,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *fo.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
|
||||
+16
-1
@@ -31,6 +31,8 @@ type IamOptions struct {
|
||||
filer *string
|
||||
masters *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
selfServiceRequiresGrant *bool
|
||||
}
|
||||
@@ -40,6 +42,7 @@ func init() {
|
||||
iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
|
||||
iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
|
||||
iamStandaloneOptions.ipBind = cmdIam.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
|
||||
iamStandaloneOptions.selfServiceRequiresGrant = cmdIam.Flag.Bool("selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
}
|
||||
@@ -119,8 +122,20 @@ func (iamopt *IamOptions) startIamServer() bool {
|
||||
iamApiServer.Shutdown()
|
||||
})
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset and
|
||||
// the bind address falls back to -ip. An explicit -ip.bind pins the port to
|
||||
// exactly that address.
|
||||
bindIp := *iamopt.ip
|
||||
listenOnLocalhost := iamopt.ipBind == nil || *iamopt.ipBind == ""
|
||||
if iamopt.ipBind != nil && *iamopt.ipBind != "" {
|
||||
bindIp = *iamopt.ipBind
|
||||
}
|
||||
if iamopt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *iamopt.listenOnLocalhost
|
||||
}
|
||||
|
||||
listenAddress := fmt.Sprintf(":%d", *iamopt.port)
|
||||
iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
|
||||
iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(bindIp, *iamopt.port, time.Duration(10)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
|
||||
+30
-11
@@ -50,6 +50,7 @@ type MasterOptions struct {
|
||||
portGrpc *int
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
metaFolder *string
|
||||
peers *string
|
||||
mastersDeprecated *string // deprecated, for backward compatibility in master.follower
|
||||
@@ -81,6 +82,15 @@ type MasterOptions struct {
|
||||
// is cancelled. Used by integration tests and by weed mini; nil for
|
||||
// standalone weed master.
|
||||
shutdownCtx context.Context
|
||||
// releaseAllocOnFirstReport lifts the post-mint allocation hold as soon as
|
||||
// one full volume-server inventory has been applied. weed mini sets it in
|
||||
// single-master mode, with singleReporterAddr pinning the embedded volume
|
||||
// server as the only admitted reporter: any other volume server is refused,
|
||||
// so the warmup window after a fresh topology id is dead time.
|
||||
// Standalone weed master / weed server can face late-arriving veterans and
|
||||
// must keep the window.
|
||||
releaseAllocOnFirstReport bool
|
||||
singleReporterAddr string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -191,6 +201,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
if *masterOption.portGrpc == 0 {
|
||||
*masterOption.portGrpc = 10000 + *masterOption.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset
|
||||
// and the bind address falls back to -ip. An explicit -ip.bind pins each
|
||||
// port to exactly that address.
|
||||
listenOnLocalhost := *masterOption.ipBind == ""
|
||||
if masterOption.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *masterOption.listenOnLocalhost
|
||||
}
|
||||
if *masterOption.ipBind == "" {
|
||||
*masterOption.ipBind = *masterOption.ip
|
||||
}
|
||||
@@ -207,7 +224,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
|
||||
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Master startup error: %v", e)
|
||||
}
|
||||
@@ -264,7 +281,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
// starting grpc server
|
||||
grpcPort := *masterOption.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
@@ -480,14 +497,16 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
|
||||
VacuumIntervalSeconds: *m.vacuumIntervalSeconds,
|
||||
VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds,
|
||||
// PulseSeconds: *m.pulseSeconds,
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
ReleaseAllocOnFirstReport: m.releaseAllocOnFirstReport,
|
||||
SingleVolumeReporter: m.singleReporterAddr,
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,10 @@ func startMasterFollower(masterOptions MasterOptions) {
|
||||
option := masterOptions.toMasterOption(nil)
|
||||
option.IsFollower = true
|
||||
|
||||
listenOnLocalhost := *masterOptions.ipBind == ""
|
||||
if masterOptions.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *masterOptions.listenOnLocalhost
|
||||
}
|
||||
if *masterOptions.ipBind == "" {
|
||||
*masterOptions.ipBind = *masterOptions.ip
|
||||
}
|
||||
@@ -137,14 +141,14 @@ func startMasterFollower(masterOptions MasterOptions) {
|
||||
ms := weed_server.NewMasterServer(r, option, masters)
|
||||
listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", version.Version(), listeningAddress)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
|
||||
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Master startup error: %v", e)
|
||||
}
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *masterOptions.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
|
||||
+127
-65
@@ -821,10 +821,20 @@ func isPortAvailable(port int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// isPortUsableOnIP reports whether port can be bound on ip. While mini also
|
||||
// opens a loopback listener, the port must additionally be free on every
|
||||
// interface so that listener cannot collide with a holder on another address.
|
||||
func isPortUsableOnIP(ip string, port int, listenOnLocalhost bool) bool {
|
||||
if !isPortOpenOnIP(ip, port) {
|
||||
return false
|
||||
}
|
||||
return !listenOnLocalhost || isPortAvailable(port)
|
||||
}
|
||||
|
||||
// findAvailablePortOnIP finds the next available port on a specific IP starting from the given port
|
||||
// It skips any ports that are in the reservedPorts map (for gRPC port collision avoidance)
|
||||
// It returns the first available port found within maxAttempts, or 0 if none found
|
||||
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
|
||||
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool, listenOnLocalhost bool) int {
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
port := startPort + i
|
||||
if port > 65535 {
|
||||
@@ -836,7 +846,7 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
continue
|
||||
}
|
||||
// Check on both the specific IP and on all interfaces for maximum reliability
|
||||
if isPortOpenOnIP(ip, port) && isPortAvailable(port) {
|
||||
if isPortUsableOnIP(ip, port, listenOnLocalhost) {
|
||||
return port
|
||||
}
|
||||
}
|
||||
@@ -847,7 +857,7 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
// ensurePortAvailableOnIP ensures a port pointer points to an available port on a specific IP
|
||||
// If the port is not available, it finds the next available port and updates the pointer
|
||||
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string, listenOnLocalhost bool) error {
|
||||
// Check if this port was explicitly specified by the user (from CLI or env, before the config file was applied)
|
||||
isExplicitPort := explicitFlags[flagName]
|
||||
|
||||
@@ -863,7 +873,7 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
return fmt.Errorf("port %d for %s (specified by flag %s) is reserved for gRPC calculation and cannot be used", original, serviceName, flagName)
|
||||
}
|
||||
glog.Warningf("Port %d for %s is reserved for gRPC calculation, finding alternative...", original, serviceName)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
|
||||
} else {
|
||||
@@ -874,14 +884,14 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
}
|
||||
|
||||
// Check on both the specific IP and on all interfaces (0.0.0.0) for maximum reliability
|
||||
if !isPortOpenOnIP(ip, original) || !isPortAvailable(original) {
|
||||
if !isPortUsableOnIP(ip, original, listenOnLocalhost) {
|
||||
// If explicitly specified, fail immediately with the originally requested port
|
||||
if isExplicitPort {
|
||||
return fmt.Errorf("port %d for %s (specified by flag %s) is not available on %s and cannot be used", original, serviceName, flagName, ip)
|
||||
}
|
||||
// For default ports, try to find an alternative
|
||||
glog.Warningf("Port %d for %s is not available on %s, finding alternative port...", original, serviceName, ip)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
|
||||
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
|
||||
} else {
|
||||
@@ -897,53 +907,61 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
|
||||
// ensureAllPortsAvailableOnIP ensures all mini service ports are available on a specific IP
|
||||
// Returns an error if an explicitly specified port is unavailable.
|
||||
// This should be called before starting any services
|
||||
func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
|
||||
portConfigs := []struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{
|
||||
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc},
|
||||
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc},
|
||||
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc},
|
||||
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc, listenOnLocalhost},
|
||||
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc, listenOnLocalhost},
|
||||
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc, listenOnLocalhost},
|
||||
}
|
||||
if *miniEnableS3 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc, listenOnLocalhost})
|
||||
if miniS3Options.portIceberg != nil && *miniS3Options.portIceberg > 0 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.portIceberg, "Iceberg", "s3.port.iceberg", nil})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.portIceberg, "Iceberg", "s3.port.iceberg", nil, listenOnLocalhost})
|
||||
}
|
||||
if miniS3Options.portLance != nil && *miniS3Options.portLance > 0 {
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniS3Options.portLance, "Lance", "s3.port.lance", nil})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniS3Options.portLance, "Lance", "s3.port.lance", nil, listenOnLocalhost})
|
||||
}
|
||||
}
|
||||
// WebDAV and the Admin UI bind only -ip.bind; they never open a loopback
|
||||
// companion listener, so ports held elsewhere cannot collide with them.
|
||||
portConfigs = append(portConfigs, struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil},
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false},
|
||||
struct {
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort})
|
||||
port *int
|
||||
name string
|
||||
flagName string
|
||||
grpcPtr *int
|
||||
localhost bool
|
||||
}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
|
||||
|
||||
// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
|
||||
// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
|
||||
@@ -956,13 +974,20 @@ func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
}
|
||||
}
|
||||
|
||||
applyMiniAdminCredentialFallback(&miniAdminOptions)
|
||||
adminBindIP := miniAdminEffectiveBindIP(bindIp)
|
||||
|
||||
// Check all HTTP ports sequentially to avoid race conditions
|
||||
// Each port check and allocation must complete before the next one starts
|
||||
// to prevent multiple goroutines from claiming the same available port
|
||||
// Also avoid allocating ports that are reserved for gRPC calculation
|
||||
for _, config := range portConfigs {
|
||||
checkIP := bindIp
|
||||
if config.name == "Admin" {
|
||||
checkIP = adminBindIP
|
||||
}
|
||||
original := *config.port
|
||||
if err := ensurePortAvailableOnIP(config.port, config.name, bindIp, reservedPorts, config.flagName); err != nil {
|
||||
if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, reservedPorts, config.flagName, config.localhost); err != nil {
|
||||
return err
|
||||
}
|
||||
// If port was changed, update the reserved gRPC ports mapping
|
||||
@@ -975,7 +1000,7 @@ func ensureAllPortsAvailableOnIP(bindIp string) error {
|
||||
// Initialize all gRPC ports before services start
|
||||
// This ensures they won't be recalculated and cause conflicts
|
||||
// All gRPC port handling (calculation, validation, and assignment) is performed exclusively in initializeGrpcPortsOnIP
|
||||
allocatedPorts := initializeGrpcPortsOnIP(bindIp)
|
||||
allocatedPorts := initializeGrpcPortsOnIP(bindIp, listenOnLocalhost)
|
||||
|
||||
// Every other service binds within a moment of this check, but the admin
|
||||
// waits for all of them first. The admin gRPC port sits inside the Linux
|
||||
@@ -1054,7 +1079,7 @@ func listenMiniAdminWorker(bindIP string, port int) (net.Listener, error) {
|
||||
// This must be called after HTTP ports are finalized and before services start
|
||||
// Returns the set of ports allocated to mini services, for callers that must
|
||||
// keep optional listeners from stealing them.
|
||||
func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
func initializeGrpcPortsOnIP(bindIp string, listenOnLocalhost bool) map[int]bool {
|
||||
// Track all ports allocated (both HTTP and gRPC) to prevent collisions.
|
||||
// We must reserve HTTP ports so that gRPC fallback allocation never picks
|
||||
// a port already assigned to an HTTP service (which hasn't bound yet).
|
||||
@@ -1077,26 +1102,34 @@ func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
}
|
||||
|
||||
grpcConfigs := []struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{
|
||||
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master"},
|
||||
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer"},
|
||||
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume"},
|
||||
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master", bindIp, listenOnLocalhost},
|
||||
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer", bindIp, listenOnLocalhost},
|
||||
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume", bindIp, listenOnLocalhost},
|
||||
}
|
||||
if *miniEnableS3 {
|
||||
grpcConfigs = append(grpcConfigs, struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
}{miniS3Options.port, miniS3Options.portGrpc, "S3"})
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{miniS3Options.port, miniS3Options.portGrpc, "S3", bindIp, listenOnLocalhost})
|
||||
}
|
||||
// The Admin worker gRPC binds workerGrpcBindIp (loopback by default), not
|
||||
// -ip.bind, and never opens a companion listener.
|
||||
grpcConfigs = append(grpcConfigs, struct {
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
}{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin"})
|
||||
httpPort *int
|
||||
grpcPort *int
|
||||
name string
|
||||
ip string
|
||||
localhost bool
|
||||
}{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin", miniAdminOptions.workerGrpcBindIp, false})
|
||||
|
||||
for _, config := range grpcConfigs {
|
||||
if config.grpcPort == nil {
|
||||
@@ -1110,10 +1143,10 @@ func initializeGrpcPortsOnIP(bindIp string) map[int]bool {
|
||||
|
||||
// Verify the gRPC port is available (whether calculated or explicitly set)
|
||||
// Check on both specific IP and all interfaces, and check against already allocated ports
|
||||
if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) || allocatedPorts[*config.grpcPort] {
|
||||
if !isPortUsableOnIP(config.ip, *config.grpcPort, config.localhost) || allocatedPorts[*config.grpcPort] {
|
||||
glog.Warningf("gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name)
|
||||
originalPort := *config.grpcPort
|
||||
newPort := findAvailablePortOnIP(bindIp, originalPort+1, 100, allocatedPorts)
|
||||
newPort := findAvailablePortOnIP(config.ip, originalPort+1, 100, allocatedPorts, config.localhost)
|
||||
if newPort == 0 {
|
||||
glog.Errorf("Could not find available gRPC port for %s starting from %d, will use %d and fail on binding", config.name, originalPort+1, originalPort)
|
||||
} else {
|
||||
@@ -1404,8 +1437,14 @@ func runMini(cmd *Command, args []string) bool {
|
||||
miniAdminOptions.workerGrpcBindIp = "127.0.0.1"
|
||||
}
|
||||
|
||||
// Services keep the loopback companion listener only while -ip.bind is
|
||||
// unset and the bind address falls back to -ip. While it is on, each
|
||||
// service port must be free on every interface, not just bindIp. This
|
||||
// must be captured before *miniBindIp is filled in below.
|
||||
listenOnLocalhost := *miniBindIp == "" && !util.IsLoopbackOrWildcardAddress(bindIp)
|
||||
|
||||
// Ensure all ports are available, find alternatives if needed
|
||||
if err := ensureAllPortsAvailableOnIP(bindIp); err != nil {
|
||||
if err := ensureAllPortsAvailableOnIP(bindIp, listenOnLocalhost); err != nil {
|
||||
glog.Errorf("Port allocation failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -1430,14 +1469,18 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// ip address
|
||||
miniMasterOptions.ip = miniIp
|
||||
miniMasterOptions.ipBind = miniBindIp
|
||||
miniMasterOptions.listenOnLocalhost = &listenOnLocalhost
|
||||
miniFilerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
|
||||
miniFilerOptions.ip = miniIp
|
||||
miniFilerOptions.bindIp = miniBindIp
|
||||
miniFilerOptions.listenOnLocalhost = &listenOnLocalhost
|
||||
miniS3Options.ip = miniIp
|
||||
miniS3Options.bindIp = miniBindIp
|
||||
miniS3Options.listenOnLocalhost = &listenOnLocalhost
|
||||
miniWebDavOptions.ipBind = miniBindIp
|
||||
miniOptions.v.ip = miniIp
|
||||
miniOptions.v.bindIp = miniBindIp
|
||||
miniOptions.v.listenOnLocalhost = &listenOnLocalhost
|
||||
miniOptions.v.masters = pb.ServerAddresses(actualPeersForComponents).ToAddresses()
|
||||
miniOptions.v.idleConnectionTimeout = miniTimeout
|
||||
// Admin server binds to the same address as the other mini services.
|
||||
@@ -1521,6 +1564,22 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// tear those services down too. On Ctrl+C they rely on their own
|
||||
// OnInterrupt hooks (see grace's LIFO ordering).
|
||||
miniMasterOptions.shutdownCtx = MiniClusterCtx
|
||||
// In single-master mode mini's embedded volume server is the only reporter
|
||||
// of prior cluster state; its first heartbeat already carries the full
|
||||
// inventory, so the post-mint allocation hold ends there instead of
|
||||
// running the whole warmup window on every fresh start. The master pins
|
||||
// the embedded address and refuses any other volume server, keeping the
|
||||
// single-reporter assumption true. Multi-master minis can field
|
||||
// late-reporting volume servers and keep the regular hold.
|
||||
// Both fields are re-derived on every run: the package-level options
|
||||
// survive in-process reruns, so a multi-master mini after a
|
||||
// single-master one must not inherit the release or the stale pin.
|
||||
singleMaster := isSingleMasterMode(*miniMasterOptions.peers)
|
||||
miniMasterOptions.releaseAllocOnFirstReport = singleMaster
|
||||
miniMasterOptions.singleReporterAddr = ""
|
||||
if singleMaster {
|
||||
miniMasterOptions.singleReporterAddr = util.JoinHostPort(*miniIp, *miniOptions.v.port)
|
||||
}
|
||||
miniOptions.v.shutdownCtx = MiniClusterCtx
|
||||
miniFilerOptions.shutdownCtx = MiniClusterCtx
|
||||
// Mini is a small/dev setup with short-lived RPCs; cap the filer's
|
||||
@@ -1776,6 +1835,14 @@ func miniAdminBindIP(requestedIP string, passwordConfigured, mtlsConfigured, all
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func miniAdminEffectiveBindIP(requestedIP string) string {
|
||||
hasMTLS := util.GetViper().GetString("https.admin.key") != "" &&
|
||||
util.GetViper().GetString("https.admin.ca") != ""
|
||||
allowInsecure := miniAdminOptions.allowInsecureBind != nil &&
|
||||
*miniAdminOptions.allowInsecureBind
|
||||
return miniAdminBindIP(requestedIP, *miniAdminOptions.adminPassword != "", hasMTLS, allowInsecure)
|
||||
}
|
||||
|
||||
// miniAdminWorkerAddress encodes the finalized HTTP and gRPC ports in the
|
||||
// server-address format understood by pb.ServerToGrpcAddress.
|
||||
func miniAdminWorkerAddress(bindIP string, httpPort, grpcPort int) string {
|
||||
@@ -1833,16 +1900,11 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
applyMiniAdminCredentialFallback(&miniAdminOptions)
|
||||
|
||||
requestedBindIP := getBindIp()
|
||||
bindIP := miniAdminEffectiveBindIP(requestedBindIP)
|
||||
hasMTLS := util.GetViper().GetString("https.admin.key") != "" &&
|
||||
util.GetViper().GetString("https.admin.ca") != ""
|
||||
allowInsecure := miniAdminOptions.allowInsecureBind != nil &&
|
||||
*miniAdminOptions.allowInsecureBind
|
||||
bindIP := miniAdminBindIP(
|
||||
requestedBindIP,
|
||||
*miniAdminOptions.adminPassword != "",
|
||||
hasMTLS,
|
||||
allowInsecure,
|
||||
)
|
||||
miniAdminOptions.ip = &bindIP
|
||||
if bindIP != requestedBindIP {
|
||||
glog.Warningf(
|
||||
|
||||
@@ -17,17 +17,20 @@ var (
|
||||
)
|
||||
|
||||
type MessageQueueAgentOptions struct {
|
||||
brokers []pb.ServerAddress
|
||||
brokersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
port *int
|
||||
brokers []pb.ServerAddress
|
||||
brokersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdMqAgent.Run = runMqAgent // break init cycle
|
||||
mqAgentOptions.brokersString = cmdMqAgent.Flag.String("broker", "localhost:17777", "comma-separated message queue brokers")
|
||||
mqAgentOptions.ip = cmdMqAgent.Flag.String("ip", "", "message queue agent host address")
|
||||
mqAgentOptions.ipBind = cmdMqAgent.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
mqAgentOptions.port = cmdMqAgent.Flag.Int("port", 16777, "message queue agent gRPC server port")
|
||||
}
|
||||
|
||||
@@ -60,8 +63,20 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
|
||||
SeedBrokers: mqAgentOpt.brokers,
|
||||
}, grpcDialOption)
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset and
|
||||
// the bind address falls back to -ip. An explicit -ip.bind pins the port to
|
||||
// exactly that address.
|
||||
bindIp := *mqAgentOpt.ip
|
||||
listenOnLocalhost := mqAgentOpt.ipBind == nil || *mqAgentOpt.ipBind == ""
|
||||
if mqAgentOpt.ipBind != nil && *mqAgentOpt.ipBind != "" {
|
||||
bindIp = *mqAgentOpt.ipBind
|
||||
}
|
||||
if mqAgentOpt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *mqAgentOpt.listenOnLocalhost
|
||||
}
|
||||
|
||||
// start grpc listener
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(*mqAgentOpt.ip, *mqAgentOpt.port, 0)
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(bindIp, *mqAgentOpt.port, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *mqAgentOpt.port, err)
|
||||
}
|
||||
@@ -84,7 +99,7 @@ func (mqAgentOpt *MessageQueueAgentOptions) startQueueAgent() bool {
|
||||
}()
|
||||
}
|
||||
|
||||
glog.Infof("Start Seaweed Message Queue Agent on %s:%d", *mqAgentOpt.ip, *mqAgentOpt.port)
|
||||
glog.Infof("Start Seaweed Message Queue Agent on %s:%d", bindIp, *mqAgentOpt.port)
|
||||
if err := grpcS.Serve(grpcL); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("MQ Agent failed to start: %v", err)
|
||||
}
|
||||
|
||||
+24
-14
@@ -18,18 +18,20 @@ var (
|
||||
)
|
||||
|
||||
type MessageQueueBrokerOptions struct {
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
port *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
logFlushInterval *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
ipBind *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
logFlushInterval *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -37,6 +39,7 @@ func init() {
|
||||
mqBrokerStandaloneOptions.mastersString = cmdMqBroker.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
|
||||
mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
|
||||
mqBrokerStandaloneOptions.ipBind = cmdMqBroker.Flag.String("ip.bind", "", "ip address to bind to. If empty, listen on all interfaces.")
|
||||
mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
|
||||
mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
|
||||
@@ -94,8 +97,15 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
|
||||
glog.Fatalf("failed to create new message broker for queue server: %v", err)
|
||||
}
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset.
|
||||
// An explicit -ip.bind pins the port to exactly that address.
|
||||
listenOnLocalhost := *mqBrokerOpt.ipBind == ""
|
||||
if mqBrokerOpt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *mqBrokerOpt.listenOnLocalhost
|
||||
}
|
||||
|
||||
// start grpc listener
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
|
||||
grpcL, localL, err := util.NewIpAndLocalListeners(*mqBrokerOpt.ipBind, *mqBrokerOpt.port, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
|
||||
}
|
||||
@@ -118,7 +128,7 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
|
||||
}()
|
||||
}
|
||||
|
||||
glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ip, *mqBrokerOpt.port)
|
||||
glog.V(0).Infof("MQ Broker listening on %s:%d", *mqBrokerOpt.ipBind, *mqBrokerOpt.port)
|
||||
if err := grpcS.Serve(grpcL); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("Failed to serve MQ Broker: %v", err)
|
||||
}
|
||||
|
||||
+16
-9
@@ -46,6 +46,7 @@ type S3Options struct {
|
||||
filer *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
portHttps *int
|
||||
portGrpc *int
|
||||
@@ -359,6 +360,12 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
if *s3opt.portGrpc == 0 {
|
||||
*s3opt.portGrpc = 10000 + *s3opt.port
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset.
|
||||
// An explicit -ip.bind pins each port to exactly that address.
|
||||
listenOnLocalhost := *s3opt.bindIp == ""
|
||||
if s3opt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *s3opt.listenOnLocalhost
|
||||
}
|
||||
if *s3opt.bindIp == "" {
|
||||
*s3opt.bindIp = "0.0.0.0"
|
||||
}
|
||||
@@ -414,14 +421,14 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
// Start Iceberg REST Catalog server if enabled
|
||||
if *s3opt.portIceberg > 0 {
|
||||
if httpS := s3opt.startIcebergServer(s3ApiServer); httpS != nil {
|
||||
if httpS := s3opt.startIcebergServer(s3ApiServer, listenOnLocalhost); httpS != nil {
|
||||
httpServers = append(httpServers, httpS)
|
||||
}
|
||||
}
|
||||
|
||||
// Start Lance Namespace server if enabled
|
||||
if s3opt.portLance != nil && *s3opt.portLance > 0 {
|
||||
if httpS := s3opt.startLanceServer(s3ApiServer); httpS != nil {
|
||||
if httpS := s3opt.startLanceServer(s3ApiServer, listenOnLocalhost); httpS != nil {
|
||||
httpServers = append(httpServers, httpS)
|
||||
}
|
||||
}
|
||||
@@ -445,7 +452,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
|
||||
s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.port, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
@@ -459,7 +466,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *s3opt.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
@@ -532,7 +539,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
} else {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
|
||||
s3ApiListenerHttps, s3ApiLocalListenerHttps, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portHttps, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("S3 API HTTPS listener on %s:%d error: %v", *s3opt.bindIp, *s3opt.portHttps, err)
|
||||
}
|
||||
@@ -592,7 +599,7 @@ func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiSer
|
||||
}
|
||||
|
||||
// startIcebergServer starts the Iceberg REST Catalog server on a separate port.
|
||||
func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http.Server {
|
||||
func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer, listenOnLocalhost bool) *http.Server {
|
||||
icebergRouter := mux.NewRouter().SkipClean(true)
|
||||
// warehouse/parent query values may legally contain ';', which Go's
|
||||
// url.ParseQuery would otherwise drop
|
||||
@@ -614,7 +621,7 @@ func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portIceberg)
|
||||
icebergListener, icebergLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portIceberg, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portIceberg, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
// The catalog is auxiliary to the S3 gateway; a port conflict must not
|
||||
// take the whole server down.
|
||||
@@ -645,7 +652,7 @@ func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer) *http
|
||||
// startLanceServer starts the Lance Namespace server on a separate port. It
|
||||
// shares the Iceberg catalog's credential role: one deployment vends table
|
||||
// credentials one way, whichever catalog the client speaks to.
|
||||
func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) *http.Server {
|
||||
func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenOnLocalhost bool) *http.Server {
|
||||
lanceRouter := mux.NewRouter().SkipClean(true)
|
||||
lanceRouter.Use(util_http.EscapeSemicolonsInQuery)
|
||||
|
||||
@@ -660,7 +667,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer) *http.S
|
||||
|
||||
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portLance)
|
||||
lanceListener, lanceLocalListener, err := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portLance, time.Duration(*s3opt.idleTimeout)*time.Second)
|
||||
*s3opt.bindIp, *s3opt.portLance, time.Duration(*s3opt.idleTimeout)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Errorf("Lance Namespace listener on %s error: %v", listenAddress, err)
|
||||
return nil
|
||||
|
||||
@@ -42,14 +42,14 @@ func TestAuxListenerBindFailureIsNonFatal(t *testing.T) {
|
||||
}
|
||||
s3a := &s3api.S3ApiServer{}
|
||||
|
||||
if s := s3opt.startIcebergServer(s3a); s != nil {
|
||||
if s := s3opt.startIcebergServer(s3a, false); s != nil {
|
||||
s.Close()
|
||||
t.Error("startIcebergServer on an occupied port should return nil")
|
||||
}
|
||||
if s3opt.icebergBound.Load() {
|
||||
t.Error("icebergBound must stay false when the listener failed to bind")
|
||||
}
|
||||
if s := s3opt.startLanceServer(s3a); s != nil {
|
||||
if s := s3opt.startLanceServer(s3a, false); s != nil {
|
||||
s.Close()
|
||||
t.Error("startLanceServer on an occupied port should return nil")
|
||||
}
|
||||
|
||||
@@ -69,6 +69,9 @@
|
||||
# volume-slot deadband for shard placement: nodes whose free volume slots differ by at
|
||||
# most this much tie and fewer live shards wins; applied times data shards (0 = strict)
|
||||
# free_slots_deadband = 0
|
||||
# shard copy concurrency. Leave unset for one in-flight copy per destination disk.
|
||||
# 0 sends shards sequentially. A positive N is a global cap, still one per disk.
|
||||
# max_shard_transfers = 4
|
||||
# max retry attempts for a failed erasure coding job
|
||||
# retry_limit = 1
|
||||
# seconds to wait between retry attempts
|
||||
|
||||
@@ -63,6 +63,11 @@ copy_other = 1 # create n x 1 = n actual volumes
|
||||
threshold = 0.9 # create threshold
|
||||
reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
|
||||
disable = false # disables volume growth if true
|
||||
# allocation_deferral = "15s" # hold the first volume id allocation after a fresh
|
||||
# topology id is minted, giving a rebuilt master's
|
||||
# veteran volume servers time to report. The hold
|
||||
# lifts early once heartbeats stay clean for two
|
||||
# pulses. "0s" disables it (e.g. test setups).
|
||||
|
||||
# configuration flags for replication
|
||||
[master.replication]
|
||||
|
||||
+38
-1
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
@@ -240,6 +241,17 @@ func init() {
|
||||
|
||||
}
|
||||
|
||||
// isServerFlagPassed checks if a specific flag was passed on the command line
|
||||
func isServerFlagPassed(name string) bool {
|
||||
found := false
|
||||
cmdServer.Flag.Visit(func(f *flag.Flag) {
|
||||
if f.Name == name {
|
||||
found = true
|
||||
}
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
func runServer(cmd *Command, args []string) bool {
|
||||
|
||||
if *serverOptions.debug {
|
||||
@@ -293,6 +305,11 @@ func runServer(cmd *Command, args []string) bool {
|
||||
actualPeersForComponents = *masterOptions.peers
|
||||
}
|
||||
|
||||
// The loopback companion listener is only kept while -ip.bind is unset and
|
||||
// the bind address falls back to -ip. Capture that here: serverBindIp is
|
||||
// repointed at serverIp below, after which the flag state is no longer
|
||||
// distinguishable.
|
||||
ipBindUnspecified := *serverBindIp == ""
|
||||
if *serverBindIp == "" {
|
||||
serverBindIp = serverIp
|
||||
}
|
||||
@@ -307,31 +324,51 @@ func runServer(cmd *Command, args []string) bool {
|
||||
// ip address
|
||||
masterOptions.ip = serverIp
|
||||
masterOptions.ipBind = serverBindIp
|
||||
masterOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
// Use actualPeersForComponents for volume/filer, not masterOptions.peers which might be "none"
|
||||
filerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
|
||||
filerOptions.ip = serverIp
|
||||
filerOptions.bindIp = serverBindIp
|
||||
filerOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
s3Options.ip = serverIp
|
||||
if *s3Options.bindIp == "" {
|
||||
s3Options.listenOnLocalhost = &ipBindUnspecified
|
||||
s3Options.bindIp = serverBindIp
|
||||
}
|
||||
if sftpOptions.bindIp == nil || *sftpOptions.bindIp == "" {
|
||||
sftpOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
sftpOptions.bindIp = serverBindIp
|
||||
}
|
||||
iamOptions.ip = serverBindIp
|
||||
iamOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
iamOptions.masters = &actualPeersForComponents
|
||||
webdavOptions.ipBind = serverBindIp
|
||||
serverOptions.v.ip = serverIp
|
||||
serverOptions.v.bindIp = serverBindIp
|
||||
serverOptions.v.listenOnLocalhost = &ipBindUnspecified
|
||||
serverOptions.v.masters = pb.ServerAddresses(actualPeersForComponents).ToAddresses()
|
||||
serverOptions.v.idleConnectionTimeout = serverTimeout
|
||||
serverOptions.v.dataCenter = serverDataCenter
|
||||
serverOptions.v.rack = serverRack
|
||||
mqBrokerOptions.ip = serverIp
|
||||
mqBrokerOptions.ipBind = serverBindIp
|
||||
mqBrokerOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
mqBrokerOptions.masters = filerOptions.masters.GetInstancesAsMap()
|
||||
mqBrokerOptions.filerGroup = filerOptions.filerGroup
|
||||
mqAgentServerOptions.ip = serverIp
|
||||
mqAgentServerOptions.brokers = pb.ServerAddresses(*mqAgentServerOptions.brokersString).ToAddresses()
|
||||
mqAgentServerOptions.ipBind = serverBindIp
|
||||
mqAgentServerOptions.listenOnLocalhost = &ipBindUnspecified
|
||||
if isServerFlagPassed("mq.agent.brokers") {
|
||||
mqAgentServerOptions.brokers = pb.ServerAddresses(*mqAgentServerOptions.brokersString).ToAddresses()
|
||||
} else {
|
||||
// Default to the embedded broker's bind address: with an explicit
|
||||
// -ip.bind the broker does not listen on localhost.
|
||||
brokerHost := *serverBindIp
|
||||
if brokerHost == "0.0.0.0" || brokerHost == "::" || brokerHost == "[::]" {
|
||||
brokerHost = "localhost"
|
||||
}
|
||||
mqAgentServerOptions.brokers = []pb.ServerAddress{pb.NewServerAddress(brokerHost, *mqBrokerOptions.port, 0)}
|
||||
}
|
||||
|
||||
// serverOptions.v.pulseSeconds = pulseSeconds
|
||||
// masterOptions.pulseSeconds = pulseSeconds
|
||||
|
||||
@@ -25,6 +25,7 @@ var (
|
||||
type SftpOptions struct {
|
||||
filer *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
port *int
|
||||
sshPrivateKey *string
|
||||
hostKeysFolder *string
|
||||
@@ -105,6 +106,12 @@ func (sftpOpt *SftpOptions) resolvePaths() {
|
||||
}
|
||||
|
||||
func (sftpOpt *SftpOptions) startSftpServer() bool {
|
||||
// The loopback companion listener is only kept while -ip.bind is unset.
|
||||
// An explicit -ip.bind pins the port to exactly that address.
|
||||
listenOnLocalhost := *sftpOpt.bindIp == ""
|
||||
if sftpOpt.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *sftpOpt.listenOnLocalhost
|
||||
}
|
||||
if *sftpOpt.bindIp == "" {
|
||||
*sftpOpt.bindIp = "0.0.0.0"
|
||||
}
|
||||
@@ -194,7 +201,7 @@ func (sftpOpt *SftpOptions) startSftpServer() bool {
|
||||
|
||||
// Start the SFTP service on TCP
|
||||
listenAddress := fmt.Sprintf("%s:%d", *sftpOpt.bindIp, *sftpOpt.port)
|
||||
sftpListener, sftpLocalListener, err := util.NewIpAndLocalListeners(*sftpOpt.bindIp, *sftpOpt.port, time.Duration(10)*time.Second)
|
||||
sftpListener, sftpLocalListener, err := util.NewIpAndLocalListeners(*sftpOpt.bindIp, *sftpOpt.port, time.Duration(10)*time.Second, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("SFTP server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
|
||||
+17
-9
@@ -45,6 +45,7 @@ type VolumeServerOptions struct {
|
||||
id *string
|
||||
publicUrl *string
|
||||
bindIp *string
|
||||
listenOnLocalhost *bool
|
||||
mastersString *string
|
||||
mserverString *string // deprecated, for backward compatibility
|
||||
masters []pb.ServerAddress
|
||||
@@ -397,6 +398,13 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
*v.ip = util.DetectedHostAddress()
|
||||
glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
|
||||
}
|
||||
// The loopback companion listener is only kept while -ip.bind is unset
|
||||
// and the bind address falls back to -ip. An explicit -ip.bind pins each
|
||||
// port to exactly that address.
|
||||
listenOnLocalhost := *v.bindIp == ""
|
||||
if v.listenOnLocalhost != nil {
|
||||
listenOnLocalhost = *v.listenOnLocalhost
|
||||
}
|
||||
if *v.bindIp == "" {
|
||||
*v.bindIp = *v.ip
|
||||
}
|
||||
@@ -485,19 +493,19 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
v.syncBatchOption(),
|
||||
)
|
||||
// starting grpc server
|
||||
grpcS := v.startGrpcService(volumeServer)
|
||||
grpcS := v.startGrpcService(volumeServer, listenOnLocalhost)
|
||||
|
||||
// starting public http server
|
||||
var publicHttpDown httpdown.Server
|
||||
if v.isSeparatedPublicPort() {
|
||||
publicHttpDown = v.startPublicHttpService(publicVolumeMux)
|
||||
publicHttpDown = v.startPublicHttpService(publicVolumeMux, listenOnLocalhost)
|
||||
if nil == publicHttpDown {
|
||||
glog.Fatalf("start public http service failed")
|
||||
}
|
||||
}
|
||||
|
||||
// starting the cluster http server
|
||||
clusterHttpServer, closeCert := v.startClusterHttpService(volumeMux)
|
||||
clusterHttpServer, closeCert := v.startClusterHttpService(volumeMux, listenOnLocalhost)
|
||||
|
||||
grace.OnReload(volumeServer.LoadNewVolumes)
|
||||
grace.OnReload(volumeServer.Reload)
|
||||
@@ -632,9 +640,9 @@ func (m multiHttpdownServer) Stop() error {
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
|
||||
func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer, listenOnLocalhost bool) *grpc.Server {
|
||||
grpcPort := *v.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*v.bindIp, grpcPort, 0)
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*v.bindIp, grpcPort, 0, listenOnLocalhost)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
@@ -653,10 +661,10 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
|
||||
return grpcS
|
||||
}
|
||||
|
||||
func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
|
||||
func (v VolumeServerOptions) startPublicHttpService(handler http.Handler, listenOnLocalhost bool) httpdown.Server {
|
||||
publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort)
|
||||
glog.V(0).Infoln("Start Seaweed volume server", version.Version(), "public at", publicListeningAddress)
|
||||
publicListener, publicLocalListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.publicPort, time.Duration(*v.idleConnectionTimeout)*time.Second)
|
||||
publicListener, publicLocalListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.publicPort, time.Duration(*v.idleConnectionTimeout)*time.Second, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Volume server listener error:%v", e)
|
||||
}
|
||||
@@ -681,7 +689,7 @@ func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdo
|
||||
// for invoking the close func on every shutdown path — both the
|
||||
// SIGTERM/grace.OnInterrupt path and the shutdownCtx path used by
|
||||
// mini/integration tests.
|
||||
func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (httpdown.Server, func()) {
|
||||
func (v VolumeServerOptions) startClusterHttpService(handler http.Handler, listenOnLocalhost bool) (httpdown.Server, func()) {
|
||||
var (
|
||||
certFile, keyFile string
|
||||
)
|
||||
@@ -692,7 +700,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http
|
||||
|
||||
listeningAddress := util.JoinHostPort(*v.bindIp, *v.port)
|
||||
glog.V(0).Infof("Start Seaweed volume server %s at %s", version.Version(), listeningAddress)
|
||||
listener, localListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.port, time.Duration(*v.idleConnectionTimeout)*time.Second)
|
||||
listener, localListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.port, time.Duration(*v.idleConnectionTimeout)*time.Second, listenOnLocalhost)
|
||||
if e != nil {
|
||||
glog.Fatalf("Volume server listener error:%v", e)
|
||||
}
|
||||
|
||||
@@ -48,6 +48,9 @@ type Entry struct {
|
||||
Remote *filer_pb.RemoteEntry
|
||||
Quota int64
|
||||
WORMEnforcedAtTsNs int64
|
||||
// CommitTsNs is the meta log event timestamp of the mutation that
|
||||
// produced this version. Zero for pre-field versions; they order by mtime.
|
||||
CommitTsNs int64
|
||||
}
|
||||
|
||||
func (entry *Entry) Size() uint64 {
|
||||
@@ -76,6 +79,8 @@ func (entry *Entry) ShallowClone() *Entry {
|
||||
newEntry.Content = entry.Content
|
||||
newEntry.Remote = entry.Remote
|
||||
newEntry.Quota = entry.Quota
|
||||
newEntry.WORMEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
|
||||
newEntry.CommitTsNs = entry.CommitTsNs
|
||||
|
||||
return newEntry
|
||||
}
|
||||
@@ -109,6 +114,7 @@ func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
|
||||
message.RemoteEntry = entry.Remote
|
||||
message.Quota = entry.Quota
|
||||
message.WormEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
|
||||
message.CommitTsNs = entry.CommitTsNs
|
||||
}
|
||||
|
||||
func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
|
||||
@@ -122,6 +128,7 @@ func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
|
||||
fsEntry.Quota = message.Quota
|
||||
fsEntry.FileSize = FileSize(message)
|
||||
fsEntry.WORMEnforcedAtTsNs = message.WormEnforcedAtTsNs
|
||||
fsEntry.CommitTsNs = message.CommitTsNs
|
||||
}
|
||||
|
||||
func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
|
||||
|
||||
+26
-9
@@ -115,12 +115,6 @@ type Filer struct {
|
||||
// overwriting the unread ledger with a partial set.
|
||||
deletionLedgerBlocked atomic.Bool
|
||||
deletionLedgerFlush chan struct{}
|
||||
// metaTsPending holds per-path commit records whose KV write failed,
|
||||
// keyed by path; ordering reads consult it until the retry lands them
|
||||
// (see meta_path_ts.go). metaTsRepairing marks paths with a live
|
||||
// repair goroutine so failures neither strand a record nor pile on.
|
||||
metaTsPending sync.Map
|
||||
metaTsRepairing sync.Map
|
||||
// flushCtx is the context metadata-log flushes append under. It stays
|
||||
// live for the process lifetime and is cancelled once during Shutdown,
|
||||
// bounding every retry - in-flight and queued alike - to one deadline.
|
||||
@@ -483,8 +477,20 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
}
|
||||
*/
|
||||
|
||||
emitted := false
|
||||
stamped := false
|
||||
// On abort, release the stamp this call made — one the entry already
|
||||
// carried is a concurrent mutation's and stays for its own event.
|
||||
defer func() {
|
||||
if !emitted && stamped {
|
||||
f.releaseEntryCommitTsNs(entry.CommitTsNs)
|
||||
}
|
||||
}()
|
||||
|
||||
if oldEntry == nil {
|
||||
f.ensureEntryInode(entry)
|
||||
f.stampEntryCommitTsNs(entry)
|
||||
stamped = true
|
||||
|
||||
glog.V(4).InfofCtx(ctx, "InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
|
||||
if err := f.Store.InsertEntry(ctx, entry); err != nil {
|
||||
@@ -517,6 +523,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
|
||||
}
|
||||
glog.V(4).InfofCtx(ctx, "UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
|
||||
f.stampEntryCommitTsNs(entry)
|
||||
stamped = true
|
||||
if err := f.UpdateEntry(ctx, oldEntry, entry, isFromOtherCluster); err != nil {
|
||||
if errors.Is(err, filer_pb.ErrExistingIsDirectory) || errors.Is(err, filer_pb.ErrExistingIsFile) {
|
||||
glog.V(2).InfofCtx(ctx, "update entry %s: %v", entry.FullPath, err)
|
||||
@@ -528,6 +536,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
}
|
||||
|
||||
f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
|
||||
emitted = true
|
||||
|
||||
f.deleteChunksIfNotNew(ctx, oldEntry, entry)
|
||||
|
||||
@@ -592,16 +601,18 @@ func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, di
|
||||
}
|
||||
|
||||
glog.V(2).InfofCtx(ctx, "create directory: %s %v", dirPath, dirEntry.Mode)
|
||||
f.stampEntryCommitTsNs(dirEntry)
|
||||
mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
|
||||
if mkdirErr != nil {
|
||||
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
|
||||
if fEntry, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound || fEntry == nil {
|
||||
glog.V(3).InfofCtx(ctx, "mkdir %s: %v", dirPath, mkdirErr)
|
||||
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
|
||||
}
|
||||
} else if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
|
||||
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
|
||||
} else {
|
||||
if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
|
||||
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
|
||||
}
|
||||
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
|
||||
}
|
||||
|
||||
} else if !dirEntry.IsDirectory() {
|
||||
@@ -621,7 +632,9 @@ func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, di
|
||||
dirEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
dirEntry.Extended[s3_constants.SeaweedFSPrefixObject] = []byte("true")
|
||||
f.stampEntryCommitTsNs(dirEntry)
|
||||
if updateErr := f.Store.UpdateEntry(ctx, dirEntry); updateErr != nil {
|
||||
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
|
||||
return fmt.Errorf("promote %s to directory: %v", dirPath, updateErr)
|
||||
}
|
||||
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
|
||||
@@ -673,7 +686,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
|
||||
narrowed := existing.ShallowClone()
|
||||
narrowed.Mode = existing.Mode&^restorableModeBits | kept
|
||||
glog.V(1).InfofCtx(ctx, "restore directory %s: narrowing %v to %v", dirPath, existing.Mode, narrowed.Mode)
|
||||
f.stampEntryCommitTsNs(narrowed)
|
||||
if err := f.UpdateEntry(ctx, existing, narrowed, false); err != nil {
|
||||
f.releaseEntryCommitTsNs(narrowed.CommitTsNs)
|
||||
return err
|
||||
}
|
||||
f.NotifyUpdateEvent(ctx, existing, narrowed, false, false, nil)
|
||||
@@ -698,7 +713,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
|
||||
UserName: attrs.UserName, GroupNames: attrs.GroupNames,
|
||||
}}
|
||||
f.ensureEntryInode(dirEntry)
|
||||
f.stampEntryCommitTsNs(dirEntry)
|
||||
if err := f.Store.InsertEntry(ctx, dirEntry); err != nil {
|
||||
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
|
||||
return fmt.Errorf("restore directory %s: %v", dirPath, err)
|
||||
}
|
||||
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, false, nil)
|
||||
|
||||
@@ -136,9 +136,11 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
|
||||
existingEntry.Attr.FileSize = uint64(remoteEntry.RemoteSize)
|
||||
existingEntry.Extended = MergeRemoteContentEncoding(remoteEntry, existingEntry.Extended)
|
||||
}
|
||||
f.stampEntryCommitTsNs(existingEntry)
|
||||
if saveErr := f.Store.UpdateEntry(persistCtx, existingEntry); saveErr != nil {
|
||||
glog.Warningf("maybeLazyListFromRemote: update %s: %v", childPath, saveErr)
|
||||
}
|
||||
f.releaseEntryCommitTsNs(existingEntry.CommitTsNs)
|
||||
} else {
|
||||
// New entry not yet in local store
|
||||
var entry *Entry
|
||||
@@ -228,7 +230,9 @@ func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullP
|
||||
dirEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))
|
||||
f.stampEntryCommitTsNs(dirEntry)
|
||||
if saveErr := f.Store.UpdateEntry(ctx, dirEntry); saveErr != nil {
|
||||
glog.Warningf("maybeLazyListFromRemote: update synced_at for %s: %v", p, saveErr)
|
||||
}
|
||||
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
|
||||
}
|
||||
@@ -29,6 +29,8 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
|
||||
func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
|
||||
if metadataEventsSuppressed(ctx) {
|
||||
// The entry's stamp stays in flight for the covering event the
|
||||
// caller emits itself (the rename handler does).
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -44,6 +46,10 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
// println("fullpath:", fullpath)
|
||||
|
||||
if strings.HasPrefix(fullpath, SystemLogDir) {
|
||||
// No event covers this write: release its stamp.
|
||||
if newEntry != nil {
|
||||
f.metaLogInflight.release(newEntry.CommitTsNs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
foundSelf := false
|
||||
@@ -56,7 +62,7 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
signatures = append(signatures, f.Signature)
|
||||
}
|
||||
|
||||
event := f.newMetadataEvent(oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
|
||||
event := f.newMetadataEvent(oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures, f.stampOf(newEntry))
|
||||
// Clear the stamp after the buffer append below - deliberately also on
|
||||
// append failure (see the metaLogInflight comment).
|
||||
defer f.metaLogInflight.done(event.TsNs)
|
||||
@@ -71,10 +77,6 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
}
|
||||
|
||||
f.logMetaEvent(ctx, event)
|
||||
// Record the commit timestamp so a peer replay ordered by the entry lock
|
||||
// this mutation may still be under sees the local change, including as a
|
||||
// tombstone after the entry itself is gone.
|
||||
f.noteCommittedEvent(event)
|
||||
if sink := metadataEventSinkFromContext(ctx); sink != nil {
|
||||
sink.Record(event)
|
||||
}
|
||||
@@ -134,6 +136,28 @@ func (t *metaLogInflight) done(tsNs int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// inFlight reports whether tsNs is a pre-commit stamp still awaiting its
|
||||
// buffer append — how an event adopts the stamp an entry carries.
|
||||
func (t *metaLogInflight) inFlight(tsNs int64) bool {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
return t.stamped[tsNs] > 0
|
||||
}
|
||||
|
||||
// release drops a stamp whose event will never be appended.
|
||||
func (t *metaLogInflight) release(tsNs int64) {
|
||||
if tsNs == 0 {
|
||||
return
|
||||
}
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
if t.stamped[tsNs] <= 1 {
|
||||
delete(t.stamped, tsNs)
|
||||
} else {
|
||||
t.stamped[tsNs]--
|
||||
}
|
||||
}
|
||||
|
||||
// minTsNs returns the oldest in-flight stamp, or 0 when nothing is in flight.
|
||||
func (t *metaLogInflight) minTsNs() int64 {
|
||||
t.Lock()
|
||||
@@ -197,7 +221,41 @@ func (f *Filer) DoneMetaLogInflightForTest(tsNs int64) {
|
||||
f.metaLogInflight.done(tsNs)
|
||||
}
|
||||
|
||||
func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
|
||||
// stampEntryCommitTsNs assigns the mutation's event timestamp to the entry
|
||||
// pre-write so the emitted event and stored version carry the same commit
|
||||
// time. The stamp stays in flight until the event appends; a write no event
|
||||
// covers must release it via releaseEntryCommitTsNs.
|
||||
func (f *Filer) stampEntryCommitTsNs(entry *Entry) int64 {
|
||||
tsNs := f.metaLogInflight.stamp()
|
||||
entry.CommitTsNs = tsNs
|
||||
return tsNs
|
||||
}
|
||||
|
||||
// releaseEntryCommitTsNs drops a stamp no event will adopt.
|
||||
func (f *Filer) releaseEntryCommitTsNs(tsNs int64) {
|
||||
f.metaLogInflight.release(tsNs)
|
||||
}
|
||||
|
||||
// StampEntryCommitTsNs/ReleaseEntryCommitTsNs expose the lifecycle to the
|
||||
// filer server's mutation paths.
|
||||
func (f *Filer) StampEntryCommitTsNs(entry *Entry) int64 {
|
||||
return f.stampEntryCommitTsNs(entry)
|
||||
}
|
||||
|
||||
func (f *Filer) ReleaseEntryCommitTsNs(tsNs int64) {
|
||||
f.metaLogInflight.release(tsNs)
|
||||
}
|
||||
|
||||
// stampOf returns the event's timestamp: the entry's in-flight stamp, or a
|
||||
// fresh one for events with no newly stamped entry (deletes).
|
||||
func (f *Filer) stampOf(newEntry *Entry) int64 {
|
||||
if newEntry != nil && f.metaLogInflight.inFlight(newEntry.CommitTsNs) {
|
||||
return newEntry.CommitTsNs
|
||||
}
|
||||
return f.metaLogInflight.stamp()
|
||||
}
|
||||
|
||||
func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32, tsNs int64) *filer_pb.SubscribeMetadataResponse {
|
||||
if oldEntry == nil && newEntry == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -224,7 +282,7 @@ func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFrom
|
||||
Signatures: signatures,
|
||||
},
|
||||
// In flight until appended to the local log buffer (see metaLogInflight).
|
||||
TsNs: f.metaLogInflight.stamp(),
|
||||
TsNs: tsNs,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -754,10 +754,8 @@ func (ma *MetaAggregator) traversePeerMetadata(ctx context.Context, f *Filer, pe
|
||||
continue
|
||||
}
|
||||
entry := FromPbEntry(resp.Directory, resp.Entry)
|
||||
// Order on the sender's per-path commit timestamp, not the
|
||||
// entry's mtime — mtime is client-settable and does not track
|
||||
// when the peer committed the change. Peers that predate the
|
||||
// field send 0 and fall back to mtime.
|
||||
// Order on the sender's commit timestamp, not client-settable
|
||||
// mtime. Peers predating the field send 0: fall back to mtime.
|
||||
peerTsNs := resp.EntryTsNs
|
||||
if peerTsNs == 0 {
|
||||
peerTsNs = entry.Attr.Mtime.UnixNano()
|
||||
@@ -776,10 +774,9 @@ func (ma *MetaAggregator) traversePeerMetadata(ctx context.Context, f *Filer, pe
|
||||
}
|
||||
|
||||
// applyTraversedEntry merges one bootstrap entry under the path's replay
|
||||
// locks, keeping local state when its commit is newer than the sender's —
|
||||
// most importantly a delete, whose tombstone must not let the bootstrap
|
||||
// resurrect the path. An applied entry records the sender's commit time so a
|
||||
// later peer replay orders against it rather than the entry's mtime.
|
||||
// locks, keeping local state when its commit is newer than the sender's.
|
||||
// An applied entry keeps the sender's commit time on its stamp so a later
|
||||
// peer replay orders against it rather than the entry's mtime.
|
||||
func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs int64) error {
|
||||
unlock := f.lockReplayPaths(entry.FullPath, "")
|
||||
defer unlock()
|
||||
@@ -805,6 +802,7 @@ func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs
|
||||
return nil
|
||||
}
|
||||
}
|
||||
entry.CommitTsNs = peerTsNs
|
||||
if insertErr := f.Store.InsertEntry(ctx, entry); insertErr != nil {
|
||||
// Entry already exists (root dir, or partial previous bootstrap);
|
||||
// the lock kept the check above current, so the peer commit is newer.
|
||||
@@ -815,7 +813,6 @@ func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs
|
||||
return fmt.Errorf("update entry %s: %w", entry.FullPath, updateErr)
|
||||
}
|
||||
}
|
||||
f.notePathChange(entry.FullPath, peerTsNs, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
)
|
||||
|
||||
// newReplayTestFiler builds the minimum Filer Replay needs over a stub store.
|
||||
|
||||
func newReplayTestFiler(store FilerStore) *Filer {
|
||||
return &Filer{
|
||||
Store: NewFilerStoreWrapper(store),
|
||||
@@ -259,58 +260,6 @@ func TestReplayMoveChecksEachPathSeparately(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A local delete leaves a recorded tombstone, so a replayed create older than
|
||||
// the delete must not resurrect the path even though no entry remains to
|
||||
// compare against.
|
||||
func TestReplaySkipsCreateOlderThanLocalDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
// The delete removed the entry; its commit record survives.
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a locally deleted entry")
|
||||
}
|
||||
}
|
||||
|
||||
// A deleted directory covers its children: a bulk drop emits no per-child
|
||||
// events, so the tombstone on the ancestor is what a replayed child create
|
||||
// must check.
|
||||
func TestReplaySkipsCreateUnderDeletedAncestor(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
|
||||
f.notePathChange("/dir", now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "child", Content: []byte("stale")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), "/dir/child"); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a child of a deleted directory")
|
||||
}
|
||||
}
|
||||
|
||||
// A recorded commit timestamp orders replay, not the entry's mtime: a rename
|
||||
// keeps the source's mtime on the destination, and a mounted client can set
|
||||
// mtime arbitrarily, so an entry whose mtime postdates the event must still
|
||||
@@ -324,9 +273,9 @@ func TestReplayAppliesWhenCommitRecordIsOlderThanFutureMtime(t *testing.T) {
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set or kept from a rename source
|
||||
Content: []byte("old destination"),
|
||||
// The last committed change predates the event.
|
||||
CommitTsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
// The last committed change predates the event.
|
||||
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
|
||||
|
||||
event := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/a",
|
||||
@@ -372,9 +321,9 @@ func TestReplaySerializesWithEntryLock(t *testing.T) {
|
||||
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now}}
|
||||
|
||||
// A local mutation holding the path lock; a replay started behind it
|
||||
// sees the recorded commit and skips.
|
||||
// sees the entry's commit stamp and skips.
|
||||
held := f.EntryLockTable.AcquireLock("test local write", path, util.ExclusiveLock)
|
||||
f.notePathChange(path, now.UnixNano(), false)
|
||||
store.entries[string(path)].CommitTsNs = now.UnixNano()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
@@ -421,18 +370,6 @@ func TestReplicateMetadataChangeAbortsOnStop(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Commit-record keys must stay small: SQL KV stores keep the key suffix in a
|
||||
// VARCHAR(766) column, so a raw path over ~578 bytes silently drops the
|
||||
// record (the insert fails and the fallback update touches no row).
|
||||
func TestMetaPathKeyFixedSizeForLongPaths(t *testing.T) {
|
||||
path := util.NewFullPath("/", string(make([]rune, 600)))
|
||||
for _, p := range []util.FullPath{"/a", path} {
|
||||
if got := len(metaPathKey(metaPathTsPrefix, p)); got != len(metaPathTsPrefix)+16 {
|
||||
t.Fatalf("metaPathKey(%d-byte path) = %d bytes, want %d", len(p), got, len(metaPathTsPrefix)+16)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exact-path locking alone lets a replayed child create slip between a local
|
||||
// ancestor delete's tombstone check and the sweep: replay locks every
|
||||
// ancestor too, so it waits for the delete's lock and then sees the
|
||||
@@ -444,7 +381,6 @@ func TestReplaySerializesWithAncestorLock(t *testing.T) {
|
||||
now := time.Now()
|
||||
child := util.NewFullPath("/dir", "file")
|
||||
held := f.EntryLockTable.AcquireLock("test local delete", "/dir", util.ExclusiveLock)
|
||||
f.notePathChange("/dir", now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
@@ -466,32 +402,9 @@ func TestReplaySerializesWithAncestorLock(t *testing.T) {
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), child); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a child of a locally deleted directory")
|
||||
}
|
||||
}
|
||||
|
||||
// Bootstrap ordering uses the sender's commit timestamp: a local commit
|
||||
// newer than the traversed entry keeps its state even when the peer entry's
|
||||
// mtime is far in the future.
|
||||
func TestBootstrapSkipsPeerEntryBehindLocalDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // client-settable, must not win
|
||||
Content: []byte("stale peer entry"),
|
||||
}
|
||||
if err := f.applyTraversedEntry(context.Background(), entry, now.Add(-time.Minute).UnixNano()); err != nil {
|
||||
t.Fatalf("applyTraversedEntry: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("bootstrap resurrected a path newer local delete covered")
|
||||
// No tombstone protects the deleted ancestor: the stale create lands.
|
||||
if _, err := store.FindEntry(context.Background(), child); err != nil {
|
||||
t.Fatalf("stale create applies once the lock releases: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -555,11 +468,11 @@ func TestBootstrapUpdatesExistingByCommitTime(t *testing.T) {
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set
|
||||
Content: []byte("old local write"),
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set
|
||||
Content: []byte("old local write"),
|
||||
CommitTsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
@@ -612,139 +525,23 @@ func TestBootstrapCopiesMissingEntryWithoutCommitTime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A delete whose commit record failed to persist still blocks a stale
|
||||
// replayed create: the pending record stands in until the retry lands it.
|
||||
func TestPendingDeleteRecordBlocksStaleReplay(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
// The delete already committed; only its ordering record failed.
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("stale create resurrected a path whose delete record is still pending")
|
||||
}
|
||||
}
|
||||
|
||||
// The failed record write retries until it lands, then leaves the pending set.
|
||||
func TestPendingCommitRecordRepairs(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f.notePathChange(path, 100, true)
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
t.Fatal("failed record write must stay pending")
|
||||
}
|
||||
store.setKvPutErr(nil)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path))
|
||||
if err != nil || !found || ts != 100 {
|
||||
t.Fatalf("delete record after repair: ts=%v found=%v err=%v", ts, found, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("pending record was not repaired")
|
||||
}
|
||||
|
||||
// CommitTsNs reports the recorded commit time over the entry mtime, and a
|
||||
// still-pending record over both.
|
||||
func TestCommitTsNsPrefersRecordAndPending(t *testing.T) {
|
||||
// CommitTsNs reports the commit timestamp stamped on the entry, or the
|
||||
// fallback when the entry predates stamping or is gone.
|
||||
func TestCommitTsNsFromEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 7 {
|
||||
t.Fatalf("no record: CommitTsNs = %d, %v; want fallback 7", got, err)
|
||||
t.Fatalf("no entry: CommitTsNs = %d, %v; want fallback 7", got, err)
|
||||
}
|
||||
f.notePathChange(path, 50, false)
|
||||
store.entries[string(path)] = &Entry{FullPath: path, CommitTsNs: 50}
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 50 {
|
||||
t.Fatalf("recorded: CommitTsNs = %d, %v; want 50", got, err)
|
||||
t.Fatalf("stamped: CommitTsNs = %d, %v; want 50", got, err)
|
||||
}
|
||||
pending := &metaTsPendingRec{}
|
||||
pending.changeTsNs.Store(80)
|
||||
f.metaTsPending.Store(string(path), pending)
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 80 {
|
||||
t.Fatalf("pending: CommitTsNs = %d, %v; want 80", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A successful write discharges what the pending record owed before it, so a
|
||||
// repair pass can never put an older timestamp back over it.
|
||||
func TestPendingCommitRecordDischargedByNewerWrite(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f.notePathChange(path, 100, false)
|
||||
store.setKvPutErr(nil)
|
||||
f.notePathChange(path, 200, false)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path))
|
||||
if err != nil || !found || ts != 200 {
|
||||
t.Fatalf("MpTs after repair = %v, %v, %v; want 200", ts, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed delete record keeps its own time even when a later failed change
|
||||
// merges into the pending record: the repair must not stamp the delete at
|
||||
// the newer change's time.
|
||||
func TestPendingDeleteRecordKeepsItsTime(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
f.notePathChange(path, 100, true)
|
||||
f.notePathChange(path, 200, false)
|
||||
store.setKvPutErr(nil)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path)); err != nil || !found || ts != 200 {
|
||||
t.Fatalf("MpTs = %v, %v, %v; want 200", ts, found, err)
|
||||
}
|
||||
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path)); err != nil || !found || ts != 100 {
|
||||
t.Fatalf("MpDl = %v, %v, %v; want 100", ts, found, err)
|
||||
store.entries[string(path)].CommitTsNs = 0
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 7 {
|
||||
t.Fatalf("unstamped: CommitTsNs = %d, %v; want fallback 7", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -906,9 +703,10 @@ func TestReplayDirectoryDeleteKeepsNewerChildren(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A directory kept for newer descendants still records the delete: a stale
|
||||
// create replayed later for another child must be fenced by it.
|
||||
func TestReplayKeptDirectoryDeleteFencesStaleCreates(t *testing.T) {
|
||||
// A directory kept for newer descendants records no delete — tombstones are
|
||||
// gone — so a stale create replayed later for another child resurrects
|
||||
// under it. This documents the accepted tradeoff.
|
||||
func TestReplayKeptDirectoryDeleteAllowsStaleCreates(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
@@ -952,8 +750,8 @@ func TestReplayKeptDirectoryDeleteFencesStaleCreates(t *testing.T) {
|
||||
if err := f.Replay(stale); err != nil {
|
||||
t.Fatalf("replay stale create: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(ctx, "/d/late.txt"); !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
t.Fatal("stale create resurrected a child under a directory the origin deleted")
|
||||
if _, err := store.FindEntry(ctx, "/d/late.txt"); err != nil {
|
||||
t.Fatalf("no tombstone exists, so the stale create lands: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1569,3 +1367,45 @@ func TestReplaySweepKeepsRemoteOnlyOwnerOfPath(t *testing.T) {
|
||||
t.Fatalf("the owned remote object must be kept, got %v", stub.deleteCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// A directory metadata update is not a subtree delete: replaying one must
|
||||
// leave no tombstone, or a child event older than the update is suppressed
|
||||
// even though nobody deleted the children.
|
||||
func TestReplayDirUpdateDoesNotSuppressChildren(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
|
||||
store.entries["/dir"] = &Entry{
|
||||
FullPath: "/dir",
|
||||
Attr: Attr{Mtime: now.Add(-time.Hour), Mode: os.ModeDir | 0755},
|
||||
}
|
||||
update := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
|
||||
NewEntry: &filer_pb.Entry{Name: "dir", IsDirectory: true},
|
||||
},
|
||||
TsNs: now.UnixNano(),
|
||||
}
|
||||
if err := f.Replay(update); err != nil {
|
||||
t.Fatalf("Replay dir update: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), "/dir"); err != nil {
|
||||
t.Fatalf("the updated directory must be back in place: %v", err)
|
||||
}
|
||||
|
||||
create := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "child", Content: []byte("data")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(create); err != nil {
|
||||
t.Fatalf("Replay child create: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), "/dir/child"); err != nil {
|
||||
t.Fatalf("a child create older than the dir update must apply: %v", err)
|
||||
}
|
||||
}
|
||||
+72
-233
@@ -5,35 +5,28 @@ import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
// metaPathTsPrefix keys per-path commit timestamps in the filer store KV:
|
||||
// the newest event timestamp this filer committed or applied for the
|
||||
// path. Written for every emitted mutation event and kept after the entry
|
||||
// is deleted, so a replayed peer event can still be ordered against a
|
||||
// change that left no entry behind.
|
||||
// metaPathTsPrefix keys the per-path commit timestamps older builds
|
||||
// wrote into the store KV for every mutation and kept after the entry
|
||||
// was deleted. They are no longer written — ordering lives on the
|
||||
// entry's CommitTsNs — but records left behind by an upgrade still
|
||||
// fence replayed events older than the change they recorded.
|
||||
metaPathTsPrefix = "MpTs"
|
||||
// metaPathDelPrefix keys delete-only timestamps. A replayed event checks
|
||||
// the path's ancestors too, so a directory delete also covers children
|
||||
// swept without per-child events (a dropped bucket's whole-store delete,
|
||||
// or a walk aborted partway).
|
||||
// metaPathDelPrefix keys the delete timestamps those builds kept. The
|
||||
// replay check consults the path's ancestors too, so a directory's
|
||||
// retained delete record covers children deleted without per-child
|
||||
// events.
|
||||
metaPathDelPrefix = "MpDl"
|
||||
)
|
||||
|
||||
// metaPathKey hashes the path into a fixed-size key. SQL-backed KV stores
|
||||
// split the key at byte 8 and store the remainder base64-encoded in a
|
||||
// VARCHAR(766) name column: a raw path over ~578 bytes overflows it, the
|
||||
// KvPut insert fails, and the fallback update touches no row yet returns no
|
||||
// error — the commit record would never land and stale replay could restore
|
||||
// the path.
|
||||
// metaPathKey hashes the path into the fixed-size key the writing builds
|
||||
// used: SQL stores split the key at byte 8 and base64 the rest into a
|
||||
// bounded name column, which a raw path can overflow.
|
||||
func metaPathKey(prefix string, path util.FullPath) []byte {
|
||||
sum := sha256.Sum256([]byte(path))
|
||||
key := make([]byte, len(prefix)+16)
|
||||
@@ -59,212 +52,65 @@ func (f *Filer) kvReadTsNs(ctx context.Context, key []byte) (tsNs int64, found b
|
||||
return int64(util.BytesToUint64(value)), true, nil
|
||||
}
|
||||
|
||||
// notePathChange persists tsNs as path's newest committed change. Local event
|
||||
// timestamps are stamped monotonically (metaLogInflight.stamp), so a plain
|
||||
// put cannot regress an earlier local record; a higher peer record it may
|
||||
// overwrite was already newer under last-writer-wins ordering anyway. A
|
||||
// delete additionally writes a delete record the ancestor lookup consults.
|
||||
//
|
||||
// The write uses a fresh context: callers notify after CommitTransaction,
|
||||
// and SQL stores keep the already-committed *sql.Tx in the request context —
|
||||
// a record written through it would fail and the path would lose ordering.
|
||||
//
|
||||
// A failed write is kept in metaTsPending and retried in the background: the
|
||||
// mutation the record orders has already committed, so dropping it would let
|
||||
// a stale replayed change overwrite or resurrect the path while the store is
|
||||
// unreachable. Ordering reads consult the pending record until the retry
|
||||
// lands it.
|
||||
func (f *Filer) notePathChange(path util.FullPath, tsNs int64, deleted bool) {
|
||||
if f.Store == nil {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
key := string(path)
|
||||
rec := f.pendingRec(key)
|
||||
defer rec.mu.Unlock()
|
||||
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if err := f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); err != nil {
|
||||
glog.WarningfCtx(ctx, "record meta change time %s: %v", path, err)
|
||||
if tsNs > rec.changeTsNs.Load() {
|
||||
rec.changeTsNs.Store(tsNs)
|
||||
}
|
||||
} else if rec.changeTsNs.Load() <= tsNs {
|
||||
// This write covers every change record owed so far.
|
||||
rec.changeTsNs.Store(0)
|
||||
}
|
||||
if deleted {
|
||||
if err := f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); err != nil {
|
||||
glog.WarningfCtx(ctx, "record meta delete time %s: %v", path, err)
|
||||
if tsNs > rec.deleteTsNs.Load() {
|
||||
rec.deleteTsNs.Store(tsNs)
|
||||
}
|
||||
} else if rec.deleteTsNs.Load() <= tsNs {
|
||||
rec.deleteTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
if rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0 {
|
||||
f.metaTsPending.CompareAndDelete(key, rec)
|
||||
return
|
||||
}
|
||||
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
|
||||
go f.repairMetaPathTs(path)
|
||||
}
|
||||
}
|
||||
|
||||
// metaTsPendingRec is a commit record still on its way to the store, holding
|
||||
// only work not yet written: changeTsNs is the newest committed change the
|
||||
// MpTs record lacks and deleteTsNs the newest delete the MpDl record lacks —
|
||||
// the two keep their own times so a later update cannot masquerade as the
|
||||
// delete. Store writes and the bookkeeping they discharge share mu, so a
|
||||
// repair pass can never write an older record over a newer one that already
|
||||
// landed; readers take the atomic fields directly.
|
||||
type metaTsPendingRec struct {
|
||||
mu sync.Mutex
|
||||
changeTsNs atomic.Int64
|
||||
deleteTsNs atomic.Int64
|
||||
}
|
||||
|
||||
// pendingRec returns the live pending record for key, locked for mutation,
|
||||
// recreating it when a repair already dropped the previous one.
|
||||
func (f *Filer) pendingRec(key string) *metaTsPendingRec {
|
||||
for {
|
||||
cur, _ := f.metaTsPending.LoadOrStore(key, &metaTsPendingRec{})
|
||||
rec := cur.(*metaTsPendingRec)
|
||||
rec.mu.Lock()
|
||||
if live, ok := f.metaTsPending.Load(key); ok && live == cur {
|
||||
return rec
|
||||
}
|
||||
rec.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// repairMetaPathTs retries a pending commit record until it lands. Each pass
|
||||
// writes only what the record still owes, so a merge by a later failure is
|
||||
// picked up without another goroutine and a landed write is never rewritten.
|
||||
// The retry is unbounded — the mutation is already committed and only the
|
||||
// record preserves its ordering.
|
||||
func (f *Filer) repairMetaPathTs(path util.FullPath) {
|
||||
key := string(path)
|
||||
ctx := f.flushCtx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
defer func() {
|
||||
f.metaTsRepairing.Delete(key)
|
||||
// A failure landing between the last pass and the flag release
|
||||
// still needs a repair pass — unless the retry context is done.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := f.metaTsPending.Load(key); ok {
|
||||
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
|
||||
go f.repairMetaPathTs(path)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
if _, ok := f.metaTsPending.Load(key); !ok {
|
||||
return
|
||||
}
|
||||
rec := f.pendingRec(key)
|
||||
var tsErr, delErr error
|
||||
if tsNs := rec.changeTsNs.Load(); tsNs > 0 {
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if tsErr = f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); tsErr == nil {
|
||||
rec.changeTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
if tsNs := rec.deleteTsNs.Load(); tsNs > 0 {
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if delErr = f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); delErr == nil {
|
||||
rec.deleteTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
done := rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0
|
||||
if done {
|
||||
f.metaTsPending.CompareAndDelete(key, rec)
|
||||
}
|
||||
rec.mu.Unlock()
|
||||
if !done {
|
||||
glog.V(0).InfofCtx(ctx, "retry record meta change time %s: %v %v", path, tsErr, delErr)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pendingPathTsNs returns the newest pending commit timestamp for path and
|
||||
// the newest pending delete timestamp.
|
||||
func (f *Filer) pendingPathTsNs(path util.FullPath) (changeTsNs, deleteTsNs int64) {
|
||||
cur, ok := f.metaTsPending.Load(string(path))
|
||||
if !ok {
|
||||
return 0, 0
|
||||
}
|
||||
rec := cur.(*metaTsPendingRec)
|
||||
return rec.changeTsNs.Load(), rec.deleteTsNs.Load()
|
||||
}
|
||||
|
||||
// CommitTsNs returns the newest commit timestamp known for path — the
|
||||
// recorded event time, or a still-pending one — and fallbackTsNs when none
|
||||
// exists. Peers stamp traversal entries with it so a bootstrap subscriber
|
||||
// orders each entry on the sender's commit clock rather than the entry's
|
||||
// client-settable mtime. A store read failure is an error, not a fallback:
|
||||
// the mtime it would mint could outdate real commits.
|
||||
// CommitTsNs resolves the commit timestamp at path for traversal: the
|
||||
// entry's stamp, else fallbackTsNs for pre-stamp or missing entries.
|
||||
func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
|
||||
if f.Store == nil {
|
||||
return fallbackTsNs, nil
|
||||
}
|
||||
tsNs := fallbackTsNs
|
||||
entry, err := f.Store.FindEntry(ctx, path)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return f.retainedCommitTsNs(ctx, path, fallbackTsNs)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
if entry == nil {
|
||||
return f.retainedCommitTsNs(ctx, path, fallbackTsNs)
|
||||
}
|
||||
return f.EntryCommitTsNs(ctx, entry, fallbackTsNs)
|
||||
}
|
||||
|
||||
// EntryCommitTsNs returns the newest commit time known for an already-loaded
|
||||
// entry: its stamp, a retained per-path record left by an older build —
|
||||
// which can outdate a replayed peer stamp — else fallbackTsNs.
|
||||
func (f *Filer) EntryCommitTsNs(ctx context.Context, entry *Entry, fallbackTsNs int64) (int64, error) {
|
||||
tsNs := entry.CommitTsNs
|
||||
if f.Store == nil {
|
||||
if tsNs == 0 {
|
||||
return fallbackTsNs, nil
|
||||
}
|
||||
return tsNs, nil
|
||||
}
|
||||
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, entry.FullPath))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found && recTs > tsNs {
|
||||
tsNs = recTs
|
||||
}
|
||||
if tsNs == 0 {
|
||||
return fallbackTsNs, nil
|
||||
}
|
||||
return tsNs, nil
|
||||
}
|
||||
|
||||
// retainedCommitTsNs reads the per-path record an older build may have left
|
||||
// for a path that has no entry now — a delete under the recording builds
|
||||
// kept its change record past the entry's removal.
|
||||
func (f *Filer) retainedCommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
|
||||
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found {
|
||||
tsNs = recTs
|
||||
return recTs, nil
|
||||
}
|
||||
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
|
||||
tsNs = pendTs
|
||||
}
|
||||
return tsNs, nil
|
||||
return fallbackTsNs, nil
|
||||
}
|
||||
|
||||
// noteCommittedEvent records the event's commit timestamp for every path it
|
||||
// touches: the source of a delete or rename, and the target of a create,
|
||||
// update, or rename. It runs after the local mutation commits while the
|
||||
// caller still holds the path's entry lock (for filer-server mutations), so
|
||||
// a peer replay ordered by the same lock sees either the whole local change
|
||||
// or none of it.
|
||||
func (f *Filer) noteCommittedEvent(event *filer_pb.SubscribeMetadataResponse) {
|
||||
message := event.EventNotification
|
||||
if message == nil {
|
||||
return
|
||||
}
|
||||
var source, target util.FullPath
|
||||
if message.OldEntry != nil {
|
||||
source = util.FullPath(filer_pb.MetadataEventSourceFullPath(event))
|
||||
}
|
||||
if message.NewEntry != nil {
|
||||
target = util.FullPath(filer_pb.MetadataEventTargetFullPath(event))
|
||||
}
|
||||
// A delete or a rename removes the source path; an update reports the
|
||||
// same path on both sides and counts once.
|
||||
if source != "" && source != target {
|
||||
f.notePathChange(source, event.TsNs, true)
|
||||
}
|
||||
if target != "" {
|
||||
f.notePathChange(target, event.TsNs, false)
|
||||
}
|
||||
}
|
||||
|
||||
// deletedAfterTsNs returns the newest recorded delete timestamp among path
|
||||
// and its ancestors.
|
||||
// deletedAfterTsNs returns the newest retained delete record among path and
|
||||
// its ancestors.
|
||||
func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64, error) {
|
||||
var tsNs int64
|
||||
for p := string(path); p != "" && p != "/"; {
|
||||
@@ -275,9 +121,6 @@ func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64
|
||||
if found && ts > tsNs {
|
||||
tsNs = ts
|
||||
}
|
||||
if _, pendDel := f.pendingPathTsNs(util.FullPath(p)); pendDel > tsNs {
|
||||
tsNs = pendDel
|
||||
}
|
||||
i := strings.LastIndexByte(p, '/')
|
||||
if i <= 0 {
|
||||
break
|
||||
@@ -287,31 +130,27 @@ func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64
|
||||
return tsNs, nil
|
||||
}
|
||||
|
||||
// localCommitTsNs returns the newest timestamp this filer is known to have
|
||||
// committed for path: the recorded event time when one exists — it is not
|
||||
// client-controllable the way an entry's mtime is — else the entry's mtime
|
||||
// for entries written before recording existed, plus any recorded delete of
|
||||
// the path or an ancestor.
|
||||
// localCommitTsNs returns this filer's newest known commit time for path:
|
||||
// the entry's stamp — not client-controllable like mtime — else mtime, plus
|
||||
// any retained change or delete record an older build left for the path or
|
||||
// its ancestors.
|
||||
func (f *Filer) localCommitTsNs(ctx context.Context, path util.FullPath) (int64, error) {
|
||||
if f.Store == nil {
|
||||
return 0, nil
|
||||
}
|
||||
var tsNs int64
|
||||
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
|
||||
entry, err := f.Store.FindEntry(ctx, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found {
|
||||
tsNs = recTs
|
||||
} else if entry, findErr := f.Store.FindEntry(ctx, path); findErr != nil {
|
||||
if !errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
return 0, findErr
|
||||
if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return 0, err
|
||||
}
|
||||
} else if entry != nil {
|
||||
tsNs = entry.Mtime.UnixNano()
|
||||
}
|
||||
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
|
||||
tsNs = pendTs
|
||||
if entry == nil {
|
||||
if tsNs, err = f.retainedCommitTsNs(ctx, path, 0); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else if tsNs, err = f.EntryCommitTsNs(ctx, entry, entry.Mtime.UnixNano()); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
delTs, err := f.deletedAfterTsNs(ctx, path)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
)
|
||||
|
||||
// Committed entries carry their own commit timestamp; deletes write
|
||||
// nothing — no per-path record and no tombstone.
|
||||
func TestPathCommitStampedOnEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
entry := &Entry{FullPath: "/dir/file", Content: []byte("data")}
|
||||
f.StampEntryCommitTsNs(entry)
|
||||
if err := store.InsertEntry(ctx, entry); err != nil {
|
||||
t.Fatalf("InsertEntry: %v", err)
|
||||
}
|
||||
f.releaseEntryCommitTsNs(entry.CommitTsNs)
|
||||
if got, err := f.CommitTsNs(ctx, "/dir/file", 0); err != nil || got != entry.CommitTsNs {
|
||||
t.Fatalf("CommitTsNs = %d,%v; want %d", got, err, entry.CommitTsNs)
|
||||
}
|
||||
if len(store.kv) != 0 {
|
||||
t.Fatalf("kv keys written for a change event: %v", store.kv)
|
||||
}
|
||||
|
||||
if err := store.DeleteEntry(ctx, entry.FullPath); err != nil {
|
||||
t.Fatalf("DeleteEntry: %v", err)
|
||||
}
|
||||
if len(store.kv) != 0 {
|
||||
t.Fatalf("kv keys written for a change or delete: %v", store.kv)
|
||||
}
|
||||
}
|
||||
|
||||
// The emitted event's TsNs is the commit timestamp stamped on the entry
|
||||
// before its store write — identical values so peers and the local store
|
||||
// agree on last-writer-wins.
|
||||
func TestEventTsNsEqualsEntryCommitTs(t *testing.T) {
|
||||
f := &Filer{
|
||||
Signature: 7,
|
||||
LocalMetaLogBuffer: log_buffer.NewLogBuffer(
|
||||
"commit-ts-test", time.Minute,
|
||||
func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {},
|
||||
nil, nil,
|
||||
),
|
||||
}
|
||||
defer f.LocalMetaLogBuffer.ShutdownLogBuffer()
|
||||
|
||||
ctx, sink := WithMetadataEventSink(context.Background())
|
||||
newEntry := &Entry{FullPath: util.FullPath("/dir/file")}
|
||||
f.StampEntryCommitTsNs(newEntry)
|
||||
f.NotifyUpdateEvent(ctx, nil, newEntry, true, false, nil)
|
||||
|
||||
event := sink.Last()
|
||||
if event == nil {
|
||||
t.Fatal("expected metadata event")
|
||||
}
|
||||
if event.TsNs != newEntry.CommitTsNs {
|
||||
t.Fatalf("event.TsNs = %d, entry.CommitTsNs = %d; must be equal", event.TsNs, newEntry.CommitTsNs)
|
||||
}
|
||||
if got := f.metaLogInflight.minTsNs(); got != 0 {
|
||||
t.Fatalf("stamp still in flight after emit: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A notify that never saw a pre-stamp still emits at a fresh stamp — the
|
||||
// stamp/release pairing must hold either way.
|
||||
func TestEventWithoutPreStampStillEmits(t *testing.T) {
|
||||
f := &Filer{
|
||||
Signature: 7,
|
||||
LocalMetaLogBuffer: log_buffer.NewLogBuffer(
|
||||
"commit-ts-fresh", time.Minute,
|
||||
func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {},
|
||||
nil, nil,
|
||||
),
|
||||
}
|
||||
defer f.LocalMetaLogBuffer.ShutdownLogBuffer()
|
||||
|
||||
ctx, sink := WithMetadataEventSink(context.Background())
|
||||
f.NotifyUpdateEvent(ctx, nil, &Entry{FullPath: util.FullPath("/dir/file")}, true, false, nil)
|
||||
if sink.Last() == nil || sink.Last().TsNs == 0 {
|
||||
t.Fatal("expected stamped metadata event")
|
||||
}
|
||||
if got := f.metaLogInflight.minTsNs(); got != 0 {
|
||||
t.Fatalf("stamp leaked: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// putRecord seeds the per-path key an older build would have left.
|
||||
func putRecord(t *testing.T, store *stubFilerStore, prefix string, path util.FullPath, tsNs int64) {
|
||||
t.Helper()
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if err := store.KvPut(context.Background(), metaPathKey(prefix, path), value); err != nil {
|
||||
t.Fatalf("seed record %s%s: %v", prefix, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A retained change record still orders a path whose entry is gone: a
|
||||
// replayed create older than the record must not resurrect it.
|
||||
func TestRetainedChangeRecordFencesMissingEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
putRecord(t, store, metaPathTsPrefix, "/dir/file", 300)
|
||||
|
||||
if newer, err := f.replayBlockedByLocal(ctx, "/dir/file", 200); err != nil || !newer {
|
||||
t.Fatalf("event at 200 should lose to retained record at 300: newer=%v err=%v", newer, err)
|
||||
}
|
||||
if newer, err := f.replayBlockedByLocal(ctx, "/dir/file", 400); err != nil || newer {
|
||||
t.Fatalf("event at 400 should win over retained record at 300: newer=%v err=%v", newer, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A directory's retained delete record covers children removed without
|
||||
// per-child records — the replay check walks the ancestors.
|
||||
func TestRetainedAncestorDeleteRecordFencesChild(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
putRecord(t, store, metaPathDelPrefix, "/dir", 300)
|
||||
|
||||
if newer, err := f.replayBlockedByLocal(ctx, "/dir/sub/file", 200); err != nil || !newer {
|
||||
t.Fatalf("child event at 200 should lose to ancestor delete at 300: newer=%v err=%v", newer, err)
|
||||
}
|
||||
if newer, err := f.replayBlockedByLocal(ctx, "/dir/sub/file", 400); err != nil || newer {
|
||||
t.Fatalf("child event at 400 should win over ancestor delete at 300: newer=%v err=%v", newer, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The retained record orders the entry, not its client-settable mtime: a
|
||||
// mounted client can push mtime into the future, and the record must not
|
||||
// let that silence newer replayed events.
|
||||
func TestRetainedRecordOrdersUnstampedEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
path := util.FullPath("/dir/file")
|
||||
putRecord(t, store, metaPathTsPrefix, path, 300)
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: time.Unix(0, 999)},
|
||||
}
|
||||
|
||||
if tsNs, err := f.localCommitTsNs(ctx, path); err != nil || tsNs != 300 {
|
||||
t.Fatalf("localCommitTsNs = %d,%v; want the record's 300, not the entry's mtime", tsNs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// An entry stamped under the current build outranks any record an older
|
||||
// build left: the stamp is the newest commit.
|
||||
func TestEntryStampBeatsRetainedRecord(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
path := util.FullPath("/dir/file")
|
||||
putRecord(t, store, metaPathTsPrefix, path, 300)
|
||||
store.entries[string(path)] = &Entry{FullPath: path, CommitTsNs: 400}
|
||||
|
||||
if tsNs, err := f.localCommitTsNs(ctx, path); err != nil || tsNs != 400 {
|
||||
t.Fatalf("localCommitTsNs = %d,%v; want the entry stamp's 400", tsNs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// End to end: a replayed create older than a retained delete record leaves
|
||||
// the path absent.
|
||||
func TestReplayCreateBlockedByRetainedDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
ctx := context.Background()
|
||||
putRecord(t, store, metaPathDelPrefix, "/dir/file", 300)
|
||||
|
||||
event := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("resurrected")},
|
||||
},
|
||||
TsNs: 200,
|
||||
}
|
||||
if err := f.Replay(event); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(ctx, "/dir/file"); err == nil {
|
||||
t.Fatal("replayed create at 200 must not resurrect a path deleted at 300")
|
||||
}
|
||||
}
|
||||
+76
-28
@@ -23,12 +23,14 @@ import (
|
||||
// only when no newer local change is recorded, so a late event cannot revert
|
||||
// or resurrect entries.
|
||||
//
|
||||
// Ordering uses the per-path commit timestamp recorded for every emitted
|
||||
// mutation (meta_path_ts.go), which survives deletes as a tombstone and is
|
||||
// not client-controllable the way entry mtime is — a mounted client can set
|
||||
// mtime into the future, and a rename keeps the source file's mtime on the
|
||||
// destination entry. Entries written before recording existed fall back to
|
||||
// their mtime.
|
||||
// Ordering uses the commit timestamp stamped on the entry by its mutation
|
||||
// event (meta_path_ts.go), which is not client-controllable the way entry
|
||||
// mtime is — a mounted client can set mtime into the future, and a rename
|
||||
// keeps the source file's mtime on the destination entry. Entries written
|
||||
// before stamping existed fall back to their mtime, except where a
|
||||
// per-path record left by an older build remains: a retained change or
|
||||
// delete record outlives the entry it ordered, so a stale replayed event
|
||||
// cannot resurrect a path whose removal the record still covers.
|
||||
//
|
||||
// The check and the mutation run under the same per-path entry lock the
|
||||
// filer server's local writes hold, so they cannot be split by a concurrent
|
||||
@@ -57,21 +59,20 @@ func (f *Filer) Replay(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if err != nil || newer {
|
||||
return err
|
||||
}
|
||||
newEntry := FromPbEntry(dir, message.NewEntry)
|
||||
newEntry.CommitTsNs = resp.TsNs
|
||||
if message.OldEntry.IsDirectory {
|
||||
glog.V(4).Infof("updating %v", newPath)
|
||||
return f.updateExistingEntry(ctx, oldPath, newEntry)
|
||||
}
|
||||
glog.V(4).Infof("deleting %v", oldPath)
|
||||
if err := f.Store.DeleteEntry(ctx, oldPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// Record the commit before the insert too: if the insert fails and
|
||||
// the event is retried later, the path sits deleted in between —
|
||||
// without the record a stale peer create could resurrect it in the
|
||||
// gap. The entry still exists afterward, so no delete record.
|
||||
f.notePathChange(oldPath, resp.TsNs, false)
|
||||
glog.V(4).Infof("creating %v", newPath)
|
||||
newEntry := FromPbEntry(dir, message.NewEntry)
|
||||
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
f.notePathChange(oldPath, resp.TsNs, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -98,10 +99,6 @@ func (f *Filer) Replay(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The origin deleted the directory even when a newer descendant
|
||||
// keeps its entry here — recording the delete fences stale child
|
||||
// creates that would otherwise resurrect under it.
|
||||
f.notePathChange(oldPath, resp.TsNs, true)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,16 +110,68 @@ func (f *Filer) Replay(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !newer {
|
||||
glog.V(4).Infof("creating %v", newPath)
|
||||
newEntry := FromPbEntry(dir, message.NewEntry)
|
||||
newEntry.CommitTsNs = resp.TsNs
|
||||
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
f.notePathChange(newPath, resp.TsNs, false)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateExistingEntry applies a same-path update without deleting the row in
|
||||
// between: the path never sits absent, so a failed retry cannot leave it
|
||||
// unordered — a tombstone would instead read as a subtree delete that
|
||||
// suppresses a directory's children. UpdateEntry goes on a present row and
|
||||
// InsertEntry on an absent one (InsertEntry is not an upsert on stores
|
||||
// configured without upsert support, and UpdateEntry touches no row that is
|
||||
// not there).
|
||||
//
|
||||
// A peer filer sharing the store can still delete the row between the find
|
||||
// and the update — the UPDATE then changes zero rows yet reports success,
|
||||
// silently dropping the replay while its subscription offset advances. The
|
||||
// write is verified by re-reading the entry, and each pass re-checks the
|
||||
// recorded commit order so a newer peer change wins instead of being
|
||||
// resurrected over.
|
||||
func (f *Filer) updateExistingEntry(ctx context.Context, path util.FullPath, newEntry *Entry) error {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
if attempt > 0 {
|
||||
newer, err := f.replayBlockedByLocal(ctx, path, newEntry.CommitTsNs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if newer {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if _, findErr := f.Store.FindEntry(ctx, path); findErr == nil {
|
||||
if err := f.Store.UpdateEntry(ctx, newEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Store.FindEntry(ctx, path); err == nil {
|
||||
return nil
|
||||
} else if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
} else if errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
// An insert colliding with a racing peer create retries as an
|
||||
// update on the next pass.
|
||||
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return findErr
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("update %s: %w", path, lastErr)
|
||||
}
|
||||
|
||||
// deleteReplayedFolderChildren removes local entries under dir, and queues
|
||||
// the chunks of removed files for deletion. Children this filer committed
|
||||
// after the event stay — a newer local write wins over a delayed delete —
|
||||
@@ -263,7 +312,6 @@ func (f *Filer) deleteSweptChild(ctx context.Context, sub *Entry, message *filer
|
||||
if derr := f.Store.DeleteEntry(ctx, sub.FullPath); derr != nil {
|
||||
return false, fmt.Errorf("delete %s: %w", sub.FullPath, derr)
|
||||
}
|
||||
f.notePathChange(sub.FullPath, tsNs, true)
|
||||
// chunks of a hard-linked child belong to the shared inode record; free
|
||||
// them only once that record is gone. A directory entry can still carry
|
||||
// chunks of its own — an S3 child promotes an existing file to a
|
||||
@@ -294,19 +342,19 @@ func (f *Filer) replayBlockedByLocal(ctx context.Context, path util.FullPath, ts
|
||||
return localTsNs > tsNs, nil
|
||||
}
|
||||
|
||||
// lockReplayPaths holds the shared entry locks for every path the event
|
||||
// touches, in the same sorted order the rename handler takes them.
|
||||
// lockReplayPaths takes the per-path entry locks for the event's paths and
|
||||
// every ancestor of them, in lexicographic order — parents sort before
|
||||
// children, the same ordering the local rename handler's two-path lock uses,
|
||||
// so lock acquisition is deadlock-free across operations. Ancestors are
|
||||
// included because the replay check consults ancestor delete records: a
|
||||
// local recursive delete or directory rename holds only the ancestor's lock,
|
||||
// and without contending on it a replay could reinsert a stale child after
|
||||
// its check. Ancestors are locked shared — they only need to exclude an
|
||||
// exclusive holder, not each other, or every replay under a busy parent
|
||||
// would serialize — while a path the event actually changes stays exclusive
|
||||
// even when it is also the other path's ancestor. The root is never locked:
|
||||
// included because the replay check consults ancestor state — a retained
|
||||
// delete record an older build left, or the ancestor entry itself: a local
|
||||
// recursive delete or directory rename holds only the ancestor's lock, and
|
||||
// without contending on it a replay could reinsert a stale child into a
|
||||
// subtree being deleted after its check. Ancestors are locked shared —
|
||||
// they only need to exclude an exclusive holder, not each other, or every
|
||||
// replay under a busy parent would serialize — while a path the event
|
||||
// actually changes stays exclusive even when it is also the other path's
|
||||
// ancestor. The root is never locked:
|
||||
// it cannot be deleted, and a shared root lock would still be excluded by
|
||||
// every top-level delete.
|
||||
func (f *Filer) lockReplayPaths(oldPath, newPath util.FullPath) func() {
|
||||
|
||||
@@ -53,12 +53,9 @@ type IAMManager struct {
|
||||
oidcRetryAgain bool
|
||||
// oidcRefreshMu serializes refreshes from reading the store to handing
|
||||
// STS the result, so an older snapshot cannot replace a newer one.
|
||||
oidcRefreshMu sync.Mutex
|
||||
// oidcManagedProviders caches providers refreshOIDCProvidersFrom built,
|
||||
// keyed by record identity, so an unchanged record reuses its provider:
|
||||
// rebuilding would drop the provider's consumed-token (jti) replay cache
|
||||
// and let an already-used token be presented again. Guarded by
|
||||
// oidcRefreshMu.
|
||||
// oidcManagedProviders caches the provider built per stored record so a
|
||||
// refresh reuses unchanged providers and keeps their JWKS cache.
|
||||
oidcRefreshMu sync.Mutex
|
||||
oidcManagedProviders map[string]*oidcManagedProvider
|
||||
oidcAuditSink OIDCProviderAuditSink
|
||||
revocationStore SessionRevocationStore
|
||||
@@ -953,10 +950,6 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
|
||||
m.oidcManagedProviders = make(map[string]*oidcManagedProvider)
|
||||
}
|
||||
seen := make(map[string]struct{}, len(records))
|
||||
// Records for one issuer with identical validation settings share a single
|
||||
// provider — and with it the consumed-token (jti) replay store — so a token
|
||||
// spent through one account's record stays spent through the other's too.
|
||||
sharedProviders := make(map[string]*oidc.OIDCProvider)
|
||||
for _, rec := range records {
|
||||
if rec == nil || rec.URL == "" {
|
||||
continue
|
||||
@@ -967,25 +960,10 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
fingerprint := oidcProviderFingerprint(rec)
|
||||
configFingerprint := oidcProviderConfigFingerprint(rec)
|
||||
var provider *oidc.OIDCProvider
|
||||
cached := m.oidcManagedProviders[key]
|
||||
if shared, ok := sharedProviders[configFingerprint]; ok {
|
||||
provider = shared
|
||||
if cached != nil && cached.provider != nil && cached.provider != shared {
|
||||
// The record's own provider is superseded; its consumed
|
||||
// tokens move to the shared store so none reopen for replay.
|
||||
shared.MergeReplayStateFrom(cached.provider)
|
||||
}
|
||||
m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
|
||||
if cached != nil && cached.provider != nil && cached.provider != shared && !m.oidcProviderStillReferenced(cached.provider) {
|
||||
if err := cached.provider.Shutdown(ctx); err != nil {
|
||||
glog.Warningf("shutdown superseded OIDC provider %s: %v", rec.ARN, err)
|
||||
}
|
||||
}
|
||||
} else if cached != nil && cached.fingerprint == fingerprint {
|
||||
var provider *oidc.OIDCProvider
|
||||
if cached != nil && cached.fingerprint == fingerprint {
|
||||
provider = cached.provider
|
||||
sharedProviders[configFingerprint] = provider
|
||||
} else {
|
||||
var err error
|
||||
provider, err = buildOIDCProviderFromRecord(rec)
|
||||
@@ -994,20 +972,7 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
|
||||
glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err)
|
||||
continue
|
||||
}
|
||||
if cached != nil && cached.provider != nil {
|
||||
// The rebuild must keep rejecting tokens the replaced
|
||||
// provider consumed; carry its replay store over, then stop
|
||||
// its cleanup worker so only the new one sweeps the store —
|
||||
// unless another record still references that provider.
|
||||
provider.AdoptReplayStateFrom(cached.provider)
|
||||
}
|
||||
sharedProviders[configFingerprint] = provider
|
||||
m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
|
||||
if cached != nil && cached.provider != nil && !m.oidcProviderStillReferenced(cached.provider) {
|
||||
if err := cached.provider.Shutdown(ctx); err != nil {
|
||||
glog.Warningf("shutdown replaced OIDC provider %s: %v", rec.ARN, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Multiple records may share an issuer when each is scoped to a
|
||||
// different account; STS picks the right one at validation time
|
||||
@@ -1017,16 +982,9 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
|
||||
Provider: provider,
|
||||
})
|
||||
}
|
||||
for key, managed := range m.oidcManagedProviders {
|
||||
for key := range m.oidcManagedProviders {
|
||||
if _, ok := seen[key]; !ok {
|
||||
delete(m.oidcManagedProviders, key)
|
||||
// A removed record's provider may be shared by records that are
|
||||
// still present; shut it down only when nothing references it.
|
||||
if managed.provider != nil && !m.oidcProviderStillReferenced(managed.provider) {
|
||||
if err := managed.provider.Shutdown(ctx); err != nil {
|
||||
glog.Warningf("shutdown removed OIDC provider %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m.stsService.SetIAMManagedOIDCProviders(byIssuer)
|
||||
@@ -1040,17 +998,6 @@ type oidcManagedProvider struct {
|
||||
provider *oidc.OIDCProvider
|
||||
}
|
||||
|
||||
// oidcProviderStillReferenced reports whether any managed record still points
|
||||
// at the provider, so a shared provider is not shut down with a removed record.
|
||||
func (m *IAMManager) oidcProviderStillReferenced(provider *oidc.OIDCProvider) bool {
|
||||
for _, managed := range m.oidcManagedProviders {
|
||||
if managed.provider == provider {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// oidcProviderFingerprint identifies the record content a provider was built
|
||||
// from, so a refresh rebuilds the provider only when that content changed.
|
||||
func oidcProviderFingerprint(rec *OIDCProviderRecord) string {
|
||||
@@ -1067,21 +1014,6 @@ func oidcProviderFingerprint(rec *OIDCProviderRecord) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// oidcProviderConfigFingerprint identifies the validation settings a provider
|
||||
// is built from — everything buildOIDCProviderFromRecord consumes except the
|
||||
// record's name — so records that differ only in identity reuse one provider.
|
||||
func oidcProviderConfigFingerprint(rec *OIDCProviderRecord) string {
|
||||
key, _ := json.Marshal(&oidc.OIDCConfig{
|
||||
Issuer: rec.URL,
|
||||
ClientIDs: rec.ClientIDs,
|
||||
Thumbprints: rec.Thumbprints,
|
||||
AllowedPrincipalTagKeys: rec.AllowedPrincipalTagKeys,
|
||||
PolicyClaim: rec.PolicyClaim,
|
||||
})
|
||||
sum := sha256.Sum256(key)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// buildOIDCProviderFromRecord turns a stored record into a runtime
|
||||
// OIDCProvider. The provider name is the ARN so re-registration is
|
||||
// idempotent and never collides with static-config entries.
|
||||
|
||||
@@ -560,92 +560,47 @@ func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
|
||||
assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
|
||||
}
|
||||
|
||||
// A refresh must not rebuild a provider whose record is unchanged: rebuilding
|
||||
// would drop the provider's consumed-token (jti) replay cache and let an
|
||||
// already-used token be presented again.
|
||||
func TestRefreshReusesProviderWhileRecordIsUnchanged(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewMemoryOIDCProviderStore()
|
||||
// A refresh caused by an unrelated record must not rebuild a provider whose
|
||||
// own record is unchanged: rebuilding drops its JWKS cache, so a briefly
|
||||
// unreachable issuer would fail exchanges its cached keys still allow.
|
||||
func TestAnUnrelatedRefreshKeepsAProvidersRuntimeState(t *testing.T) {
|
||||
store := &persistentTestStore{NewMemoryOIDCProviderStore()}
|
||||
mgr := startServer(t, store)
|
||||
|
||||
arn := arnOf(t, persistTestAPIIssuer)
|
||||
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
|
||||
ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
|
||||
}))
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
first := mgr.oidcManagedProviders[arn].provider
|
||||
require.NotNil(t, first)
|
||||
createAPIProvider(t, mgr, persistTestAPIIssuer)
|
||||
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
assert.Same(t, first, mgr.oidcManagedProviders[arn].provider,
|
||||
"an unchanged record was rebuilt, dropping its replay cache")
|
||||
mgr.oidcRefreshMu.Lock()
|
||||
before := mgr.oidcManagedProviders[arn]
|
||||
mgr.oidcRefreshMu.Unlock()
|
||||
require.NotNil(t, before)
|
||||
|
||||
// A changed record must rebuild so the new configuration applies.
|
||||
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
|
||||
ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud", "aud2"},
|
||||
}))
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
rebuilt := mgr.oidcManagedProviders[arn].provider
|
||||
require.NotNil(t, rebuilt)
|
||||
assert.NotSame(t, first, rebuilt, "a changed record kept the old provider")
|
||||
createAPIProvider(t, mgr, persistTestStaticIssuer) // refreshes over an unrelated record
|
||||
|
||||
// Deleting the record drops the provider from the cache and the STS map.
|
||||
require.NoError(t, store.DeleteProvider(ctx, "", arn))
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
assert.NotContains(t, mgr.oidcManagedProviders, arn)
|
||||
assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer))
|
||||
mgr.oidcRefreshMu.Lock()
|
||||
after := mgr.oidcManagedProviders[arn]
|
||||
mgr.oidcRefreshMu.Unlock()
|
||||
require.NotNil(t, after)
|
||||
assert.Same(t, before.provider, after.provider, "an unchanged record's provider was rebuilt")
|
||||
}
|
||||
|
||||
// stsKnowsIssuerForAccount is stsKnowsIssuer through the account-scoped
|
||||
// lookup used for providers that have no global record.
|
||||
func stsKnowsIssuerForAccount(t *testing.T, mgr *IAMManager, issuer, accountID string) bool {
|
||||
t.Helper()
|
||||
tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
|
||||
"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
|
||||
}).SignedString([]byte("not-the-providers-key"))
|
||||
require.NoError(t, err)
|
||||
_, _, err = mgr.GetSTSService().ValidateWebIdentityTokenForAccount(context.Background(), tok, accountID)
|
||||
require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
|
||||
return !strings.Contains(err.Error(), "no identity provider registered")
|
||||
}
|
||||
|
||||
// Two records for one issuer with identical validation settings share a
|
||||
// single provider — and with it the consumed-token (jti) replay store — so a
|
||||
// token spent through one account's record stays spent through the other's.
|
||||
// Removing one record of the pair must not shut the shared provider down.
|
||||
func TestSameIssuerRecordsShareOneProvider(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewMemoryOIDCProviderStore()
|
||||
// A record whose validation settings change must get a new provider: the
|
||||
// cached runtime cannot keep serving settings the store no longer holds.
|
||||
func TestAChangedRecordRebuildsItsProvider(t *testing.T) {
|
||||
store := &persistentTestStore{NewMemoryOIDCProviderStore()}
|
||||
mgr := startServer(t, store)
|
||||
arn := arnOf(t, persistTestAPIIssuer)
|
||||
createAPIProvider(t, mgr, persistTestAPIIssuer)
|
||||
|
||||
arnA := arnOf(t, persistTestAPIIssuer) + "-account-a"
|
||||
arnB := arnOf(t, persistTestAPIIssuer) + "-account-b"
|
||||
for _, rec := range []*OIDCProviderRecord{
|
||||
{ARN: arnA, URL: persistTestAPIIssuer, AccountID: "111111111111", ClientIDs: []string{"aud"}},
|
||||
{ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"aud"}},
|
||||
} {
|
||||
require.NoError(t, store.StoreProvider(ctx, "", rec))
|
||||
}
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
mgr.oidcRefreshMu.Lock()
|
||||
before := mgr.oidcManagedProviders[arn].provider
|
||||
mgr.oidcRefreshMu.Unlock()
|
||||
|
||||
shared := mgr.oidcManagedProviders[arnA].provider
|
||||
require.NotNil(t, shared)
|
||||
assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
|
||||
"identical records for one issuer built separate replay stores")
|
||||
require.NoError(t, mgr.UpdateOIDCProviderThumbprints(context.Background(), arn,
|
||||
[]string{"0123456789abcdef0123456789abcdef01234567"}))
|
||||
|
||||
// Removing one record of the pair keeps the shared provider alive.
|
||||
require.NoError(t, store.DeleteProvider(ctx, "", arnA))
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
|
||||
"a removed sibling record shut down the shared provider")
|
||||
assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
|
||||
|
||||
// The remaining record diverging rebuilds its own provider.
|
||||
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
|
||||
ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"b-aud"},
|
||||
}))
|
||||
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
|
||||
assert.NotSame(t, shared, mgr.oidcManagedProviders[arnB].provider,
|
||||
"a changed record kept the shared provider")
|
||||
assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
|
||||
mgr.oidcRefreshMu.Lock()
|
||||
after := mgr.oidcManagedProviders[arn]
|
||||
mgr.oidcRefreshMu.Unlock()
|
||||
require.NotNil(t, after)
|
||||
assert.NotSame(t, before, after.provider, "a changed record kept its provider")
|
||||
}
|
||||
@@ -1,410 +0,0 @@
|
||||
package oidc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/iam/providers"
|
||||
)
|
||||
|
||||
const (
|
||||
// clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519)
|
||||
clockSkewTolerance = 5 * time.Minute
|
||||
// defaultJTICacheSize is the default maximum number of JTI entries to cache
|
||||
defaultJTICacheSize = 10000
|
||||
// defaultCleanupInterval is how often to scan for expired JTI entries
|
||||
defaultCleanupInterval = 5 * time.Minute
|
||||
// eagerCleanupMinInterval rate limits opportunistic sweeps near capacity
|
||||
eagerCleanupMinInterval = 5 * time.Second
|
||||
// tokenFingerprintPrefix marks replay-cache keys derived from the token's
|
||||
// own fingerprint, used when the token carries no usable jti claim
|
||||
tokenFingerprintPrefix = "fp:"
|
||||
)
|
||||
|
||||
// jtiEntry represents a cached JTI with metadata
|
||||
type jtiEntry struct {
|
||||
subject string
|
||||
expiresAt time.Time
|
||||
// uncounted marks an entry merged in past the capacity bound: it keeps
|
||||
// rejecting replays but does not charge the count, so merged excess can
|
||||
// neither reject new exchanges nor be double-counted when swept.
|
||||
uncounted bool
|
||||
}
|
||||
|
||||
// jtiReplayCache wraps the consumed-token store that backs JTI replay
|
||||
// protection. Each consumed token identifier is held until the token's exp
|
||||
// plus clock skew, and a second presentation of the same identifier is
|
||||
// rejected with ErrProviderTokenReplayed.
|
||||
//
|
||||
// The cache is bounded by maxSize and uses automatic TTL-based expiration:
|
||||
// a background worker sweeps expired entries, an eager sweep runs when the
|
||||
// cache nears capacity, and the consume path reclaims expired slots before
|
||||
// rejecting a fresh token for capacity.
|
||||
//
|
||||
// Thread Safety:
|
||||
// All methods are safe for concurrent use. The consume path uses atomic
|
||||
// LoadOrStore to prevent TOCTOU races.
|
||||
type jtiReplayCache struct {
|
||||
name string // owning provider name, for log messages
|
||||
|
||||
enabled atomic.Bool
|
||||
maxSize atomic.Int64
|
||||
lastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups
|
||||
|
||||
// storeMu guards the store/count pair: replay-state handoffs (adoptFrom,
|
||||
// mergeFrom) repoint them under the write lock while the consume path
|
||||
// holds a read lock, so a consume can never land in a store the handoff
|
||||
// already drained and abandoned.
|
||||
// It is always acquired before sweepMu, never while holding it.
|
||||
storeMu sync.RWMutex
|
||||
store *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys
|
||||
count *atomic.Int64 // entries in store; shared with the store
|
||||
|
||||
// mu guards the lifecycle of the cleanup worker: cleanupCancel and
|
||||
// cleanupDone. The worker itself only touches store/count, so holding
|
||||
// mu while waiting for it is safe.
|
||||
mu sync.Mutex
|
||||
cleanupCancel context.CancelFunc
|
||||
cleanupDone chan struct{} // Closed when cleanup goroutine exits
|
||||
|
||||
// sweepMu serializes cache sweeps so at most one scan runs at a time.
|
||||
sweepMu sync.Mutex
|
||||
}
|
||||
|
||||
func newJTIReplayCache(name string) *jtiReplayCache {
|
||||
return &jtiReplayCache{
|
||||
name: name,
|
||||
store: &sync.Map{},
|
||||
count: &atomic.Int64{},
|
||||
}
|
||||
}
|
||||
|
||||
// configure applies the replay-protection settings: a previous worker is
|
||||
// always stopped, whether the new configuration enables protection or
|
||||
// disables it. The consumed-token store survives re-configuration: resetting
|
||||
// it would let already-seen tokens be replayed against the refreshed provider.
|
||||
func (c *jtiReplayCache) configure(enabled bool, maxSize int64) {
|
||||
c.mu.Lock()
|
||||
c.stopWorkerLocked()
|
||||
if enabled {
|
||||
if maxSize <= 0 {
|
||||
maxSize = defaultJTICacheSize
|
||||
}
|
||||
// the bound must land before the flag: a consume that sees enabled
|
||||
// with a zero bound would reject the first token it registers
|
||||
c.maxSize.Store(maxSize)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
c.cleanupCancel = cancel
|
||||
c.cleanupDone = done
|
||||
c.enabled.Store(true)
|
||||
c.mu.Unlock()
|
||||
|
||||
go c.cleanupLoop(ctx, done, defaultCleanupInterval)
|
||||
glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", c.name, maxSize)
|
||||
} else {
|
||||
c.enabled.Store(false)
|
||||
c.mu.Unlock()
|
||||
glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", c.name)
|
||||
}
|
||||
}
|
||||
|
||||
// stopWorkerLocked cancels the current cleanup goroutine and waits briefly
|
||||
// for it to exit. Caller must hold mu.
|
||||
func (c *jtiReplayCache) stopWorkerLocked() {
|
||||
if c.cleanupCancel == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupCancel()
|
||||
select {
|
||||
case <-c.cleanupDone:
|
||||
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", c.name)
|
||||
case <-time.After(2 * time.Second):
|
||||
glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", c.name)
|
||||
}
|
||||
c.cleanupCancel = nil
|
||||
c.cleanupDone = nil
|
||||
}
|
||||
|
||||
// consume marks a token as used and rejects a second presentation of it.
|
||||
// jti is the token's jti claim (may be empty); tokenExpiresAt is its exp.
|
||||
// A no-op while protection is disabled.
|
||||
func (c *jtiReplayCache) consume(token, jti, subject string, tokenExpiresAt time.Time) error {
|
||||
if !c.enabled.Load() {
|
||||
return nil
|
||||
}
|
||||
|
||||
key := jti
|
||||
if key == "" {
|
||||
// Providers that issue no jti cannot be tracked by claim, so the
|
||||
// signed input keys the entry: a repeated presentation of the same
|
||||
// claims is a replay even when the signature bytes differ (e.g.
|
||||
// ECDSA s/N-s malleability).
|
||||
signingInput := token
|
||||
if i := strings.LastIndexByte(token, '.'); i > 0 {
|
||||
signingInput = token[:i]
|
||||
}
|
||||
sum := sha256.Sum256([]byte(signingInput))
|
||||
key = tokenFingerprintPrefix + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
if tokenExpiresAt.IsZero() {
|
||||
// a cache slot is freed only by expiry, so a token without exp
|
||||
// would hold it forever and eventually exhaust the bound
|
||||
return fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims)
|
||||
}
|
||||
expiresAt := tokenExpiresAt.Add(clockSkewTolerance)
|
||||
if !time.Now().Before(expiresAt) {
|
||||
return fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired)
|
||||
}
|
||||
|
||||
entry := &jtiEntry{
|
||||
subject: subject,
|
||||
expiresAt: expiresAt,
|
||||
}
|
||||
|
||||
// The read lock pins the store/count pair for the whole test-and-set
|
||||
// so a replay-state handoff cannot repoint them mid-check and strand
|
||||
// this consume in an abandoned store.
|
||||
c.storeMu.RLock()
|
||||
store, count := c.store, c.count
|
||||
|
||||
// Atomic test-and-set detects replays before any capacity work runs,
|
||||
// so a replayed token cannot trigger cleanup scans.
|
||||
if _, loaded := store.LoadOrStore(key, entry); loaded {
|
||||
c.storeMu.RUnlock()
|
||||
glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)",
|
||||
c.name, jtiLogKey(key))
|
||||
return fmt.Errorf("%w: jti %s has already been used",
|
||||
providers.ErrProviderTokenReplayed, jtiLogKey(key))
|
||||
}
|
||||
|
||||
// The count is incremented as part of the insert, so concurrent
|
||||
// validations cannot all slip under the limit: whoever pushes the
|
||||
// count over evicts their own entry and is rejected.
|
||||
newCount := count.Add(1)
|
||||
maxSize := c.maxSize.Load()
|
||||
if newCount > maxSize {
|
||||
// Over the bound: expired entries may still occupy slots, so
|
||||
// reclaim them with a synchronized sweep before rejecting.
|
||||
c.sweepStore(store, count)
|
||||
if count.Load() > maxSize {
|
||||
if _, loaded := store.LoadAndDelete(key); loaded {
|
||||
count.Add(-1)
|
||||
}
|
||||
c.storeMu.RUnlock()
|
||||
glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+
|
||||
"This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.",
|
||||
c.name, maxSize)
|
||||
return fmt.Errorf("JTI cache capacity exceeded")
|
||||
}
|
||||
} else if newCount >= maxSize*9/10 {
|
||||
glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup",
|
||||
c.name, (newCount*100)/maxSize, newCount, maxSize)
|
||||
go c.performEagerCleanup()
|
||||
}
|
||||
c.storeMu.RUnlock()
|
||||
|
||||
glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)",
|
||||
c.name, jtiLogKey(key), expiresAt, newCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// evictExpired removes entries whose expiry has passed and returns how
|
||||
// many it deleted. Caller must hold sweepMu.
|
||||
func (c *jtiReplayCache) evictExpired(now time.Time, store *sync.Map, count *atomic.Int64) int {
|
||||
removed := 0
|
||||
store.Range(func(key, value interface{}) bool {
|
||||
expiresAt := value.(*jtiEntry).expiresAt
|
||||
if !expiresAt.IsZero() && now.After(expiresAt) {
|
||||
// LoadAndDelete reports whether this call actually removed the
|
||||
// entry, so concurrent deletes cannot decrement the count twice.
|
||||
if v, loaded := store.LoadAndDelete(key); loaded {
|
||||
if !v.(*jtiEntry).uncounted {
|
||||
count.Add(-1)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return removed
|
||||
}
|
||||
|
||||
// state returns the cache's current store/count pair.
|
||||
func (c *jtiReplayCache) state() (*sync.Map, *atomic.Int64) {
|
||||
c.storeMu.RLock()
|
||||
defer c.storeMu.RUnlock()
|
||||
return c.store, c.count
|
||||
}
|
||||
|
||||
// sweepStore runs one serialized expiry scan over the given store. It only
|
||||
// takes sweepMu, so callers already holding storeMu may use it.
|
||||
func (c *jtiReplayCache) sweepStore(store *sync.Map, count *atomic.Int64) {
|
||||
c.sweepMu.Lock()
|
||||
defer c.sweepMu.Unlock()
|
||||
c.lastSweep.Store(time.Now().UnixNano())
|
||||
if removed := c.evictExpired(time.Now(), store, count); removed > 0 {
|
||||
glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)",
|
||||
c.name, removed, count.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// sweepExpired runs a scan that is guaranteed to complete before the
|
||||
// caller proceeds; the periodic worker uses it so the scan always covers
|
||||
// the store currently in effect, not one captured earlier.
|
||||
func (c *jtiReplayCache) sweepExpired() {
|
||||
store, count := c.state()
|
||||
c.sweepStore(store, count)
|
||||
}
|
||||
|
||||
// performEagerCleanup performs an opportunistic cleanup of expired JTI
|
||||
// entries when the cache approaches capacity. At most one sweep runs at a
|
||||
// time and no more often than eagerCleanupMinInterval, so a burst of
|
||||
// requests near the limit cannot each turn into a full-cache scan.
|
||||
func (c *jtiReplayCache) performEagerCleanup() {
|
||||
if !c.enabled.Load() {
|
||||
return
|
||||
}
|
||||
if time.Since(time.Unix(0, c.lastSweep.Load())) < eagerCleanupMinInterval {
|
||||
return
|
||||
}
|
||||
store, count := c.state()
|
||||
if !c.sweepMu.TryLock() {
|
||||
return
|
||||
}
|
||||
defer c.sweepMu.Unlock()
|
||||
c.lastSweep.Store(time.Now().UnixNano())
|
||||
|
||||
beforeCount := count.Load()
|
||||
if removed := c.evictExpired(time.Now(), store, count); removed > 0 {
|
||||
glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)",
|
||||
c.name, removed, beforeCount, count.Load())
|
||||
} else {
|
||||
glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+
|
||||
"Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.",
|
||||
c.name)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupLoop periodically removes expired JTI entries from the cache.
|
||||
// This prevents memory growth by removing JTIs after their tokens have
|
||||
// expired. Runs until ctx is cancelled; done is closed on exit.
|
||||
func (c *jtiReplayCache) cleanupLoop(ctx context.Context, done chan struct{}, interval time.Duration) {
|
||||
defer close(done)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", c.name, interval)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", c.name)
|
||||
return
|
||||
case <-ticker.C:
|
||||
c.sweepExpired()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stop gracefully stops the cleanup worker.
|
||||
// Should be called when the provider is being shut down to prevent goroutine
|
||||
// leaks.
|
||||
func (c *jtiReplayCache) stop(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
cancel, done := c.cleanupCancel, c.cleanupDone
|
||||
c.mu.Unlock()
|
||||
|
||||
if cancel == nil {
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", c.name)
|
||||
case <-ctx.Done():
|
||||
glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", c.name, ctx.Err())
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
if c.cleanupDone == done {
|
||||
c.cleanupCancel = nil
|
||||
c.cleanupDone = nil
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptFrom takes over the consumed-token store of another cache, so tokens
|
||||
// it already saw stay rejected across the rebuild. The caller must shut down
|
||||
// the replaced provider; only one cleanup worker may sweep the shared store
|
||||
// at a time. Call before the new provider serves validators; sweepMu
|
||||
// excludes its own cleanup worker during the swap.
|
||||
func (c *jtiReplayCache) adoptFrom(old *jtiReplayCache) {
|
||||
if old == nil || old == c {
|
||||
return
|
||||
}
|
||||
c.storeMu.Lock()
|
||||
c.sweepMu.Lock()
|
||||
c.store, c.count = old.store, old.count
|
||||
c.sweepMu.Unlock()
|
||||
c.storeMu.Unlock()
|
||||
}
|
||||
|
||||
// mergeFrom hands this cache's store to another cache and then folds the
|
||||
// other's consumed-token entries in, so rejections the other recorded keep
|
||||
// holding when its records move to a shared provider. Repointing first means
|
||||
// tokens the other still consumes — in-flight exchanges or ones routed to it
|
||||
// before the provider map updates — land in the shared store instead of a
|
||||
// store this merge already drained. Unlike adoptFrom it swaps nothing of its
|
||||
// own, so it is safe on a cache that is already serving validators.
|
||||
func (c *jtiReplayCache) mergeFrom(other *jtiReplayCache) {
|
||||
if other == nil || other == c {
|
||||
return
|
||||
}
|
||||
c.storeMu.RLock()
|
||||
defer c.storeMu.RUnlock()
|
||||
store, count := c.store, c.count
|
||||
|
||||
other.storeMu.Lock()
|
||||
oldStore := other.store
|
||||
if oldStore == store {
|
||||
other.storeMu.Unlock()
|
||||
return
|
||||
}
|
||||
other.store, other.count = store, count
|
||||
other.storeMu.Unlock()
|
||||
|
||||
c.sweepMu.Lock()
|
||||
defer c.sweepMu.Unlock()
|
||||
now := time.Now()
|
||||
oldStore.Range(func(key, value interface{}) bool {
|
||||
entry := value.(*jtiEntry)
|
||||
if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) {
|
||||
return true
|
||||
}
|
||||
// A consumed token is never dropped to fit the bound: merged entries
|
||||
// stay but do not charge the count, so a store filled past the bound
|
||||
// by the merge cannot reject new exchanges while it drains by expiry.
|
||||
store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true})
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// jtiLogKey redacts a token identifier for logs and error strings: a stable
|
||||
// fingerprint that still lets operators correlate repeated presentations.
|
||||
func jtiLogKey(jti string) string {
|
||||
sum := sha256.Sum256([]byte(jti))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -75,17 +75,6 @@ func (m *MockOIDCProvider) Authenticate(ctx context.Context, token string) (*pro
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
// AuthenticateSingleUse stays on the mock path: the embedded provider's
|
||||
// method would bypass the mock's test data and demand a real JWKS.
|
||||
func (m *MockOIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
|
||||
return m.Authenticate(ctx, token)
|
||||
}
|
||||
|
||||
// ValidateTokenSingleUse stays on the mock path; see AuthenticateSingleUse.
|
||||
func (m *MockOIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
|
||||
return m.ValidateToken(ctx, token)
|
||||
}
|
||||
|
||||
// ValidateToken validates tokens using test data
|
||||
func (m *MockOIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
|
||||
if !m.initialized {
|
||||
|
||||
@@ -27,20 +27,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
)
|
||||
|
||||
// OIDCProvider implements OpenID Connect authentication with JTI replay protection.
|
||||
//
|
||||
// Replay Protection:
|
||||
// The provider validates the 'jti' (JWT ID) claim to prevent token replay attacks.
|
||||
// Each JTI is stored in memory until the token expires. If a token with a duplicate
|
||||
// JTI is presented, it is rejected with ErrProviderTokenReplayed.
|
||||
//
|
||||
// The JTI cache is bounded by JTICacheMaxSize and uses automatic TTL-based expiration.
|
||||
// A token without a usable jti claim is tracked by the token's own fingerprint
|
||||
// instead, so a repeated presentation is still detected as a replay.
|
||||
//
|
||||
// Thread Safety:
|
||||
// All methods are safe for concurrent use. The JTI cache uses atomic LoadOrStore
|
||||
// operations to prevent TOCTOU race conditions.
|
||||
// OIDCProvider implements OpenID Connect authentication
|
||||
type OIDCProvider struct {
|
||||
name string
|
||||
config *OIDCConfig
|
||||
@@ -57,25 +44,6 @@ type OIDCProvider struct {
|
||||
jwksFetchedAt time.Time
|
||||
resolvedJWKSUri string
|
||||
discoveryFailed bool
|
||||
|
||||
// jti wraps the consumed-token cache behind the replay protection. It is
|
||||
// allocated by NewOIDCProvider or lazily by replayCache, and survives
|
||||
// re-initialization, so a config refresh cannot reopen already-consumed
|
||||
// tokens for replay. jtiOnce makes a zero-value OIDCProvider safe.
|
||||
jti *jtiReplayCache
|
||||
jtiOnce sync.Once
|
||||
}
|
||||
|
||||
// replayCache returns the consumed-token cache, allocating it for providers
|
||||
// built without NewOIDCProvider so the delegated operations never panic on a
|
||||
// nil cache.
|
||||
func (p *OIDCProvider) replayCache() *jtiReplayCache {
|
||||
p.jtiOnce.Do(func() {
|
||||
if p.jti == nil {
|
||||
p.jti = newJTIReplayCache(p.name)
|
||||
}
|
||||
})
|
||||
return p.jti
|
||||
}
|
||||
|
||||
// OIDCConfig holds OIDC provider configuration
|
||||
@@ -139,13 +107,6 @@ type OIDCConfig struct {
|
||||
// TLSInsecureSkipVerify controls whether to skip TLS verification.
|
||||
// WARNING: Should only be used in development/testing environments. Never use in production.
|
||||
TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty"`
|
||||
|
||||
// JTIReplayProtectionEnabled enables token replay protection via JTI tracking (default: true)
|
||||
// Use pointer to distinguish between "not set" (nil = default true) and "explicitly false"
|
||||
JTIReplayProtectionEnabled *bool `json:"jtiReplayProtectionEnabled,omitempty"`
|
||||
|
||||
// JTICacheMaxSize sets maximum number of JTI entries to cache (default: 10000)
|
||||
JTICacheMaxSize int `json:"jtiCacheMaxSize,omitempty"`
|
||||
}
|
||||
|
||||
// PrincipalTagsClaim is the AWS-defined namespace claim that carries
|
||||
@@ -358,7 +319,6 @@ func NewOIDCProvider(name string) *OIDCProvider {
|
||||
return &OIDCProvider{
|
||||
name: name,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect},
|
||||
jti: newJTIReplayCache(name),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,6 +422,9 @@ func (p *OIDCProvider) Initialize(config interface{}) error {
|
||||
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
// A replaced provider drops its transport without a close; bound
|
||||
// idle connections so they cannot linger with their goroutines.
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
p.httpClient = &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
@@ -469,17 +432,6 @@ func (p *OIDCProvider) Initialize(config interface{}) error {
|
||||
CheckRedirect: refuseDowngradeRedirect,
|
||||
}
|
||||
|
||||
// Configure JTI replay protection last, so a config failure above cannot
|
||||
// leave a running worker behind. A previous worker is always stopped,
|
||||
// whether the new configuration enables protection or disables it.
|
||||
// The consumed-token store survives re-initialization: resetting it would
|
||||
// let already-seen tokens be replayed against the refreshed provider.
|
||||
jtiEnabled := true
|
||||
if oidcConfig.JTIReplayProtectionEnabled != nil {
|
||||
jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled
|
||||
}
|
||||
p.replayCache().configure(jtiEnabled, int64(oidcConfig.JTICacheMaxSize))
|
||||
|
||||
// For testing, we'll skip the actual OIDC client initialization
|
||||
return nil
|
||||
}
|
||||
@@ -530,18 +482,6 @@ func (p *OIDCProvider) allowedAudiences() []string {
|
||||
|
||||
// Authenticate authenticates a user with an OIDC token
|
||||
func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
|
||||
return p.authenticate(ctx, token, false)
|
||||
}
|
||||
|
||||
// AuthenticateSingleUse authenticates and consumes the token so a second
|
||||
// presentation is rejected as a replay. The web-identity exchange uses it,
|
||||
// where a token buys a session exactly once; callers that validate the same
|
||||
// token on every request use Authenticate.
|
||||
func (p *OIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
|
||||
return p.authenticate(ctx, token, true)
|
||||
}
|
||||
|
||||
func (p *OIDCProvider) authenticate(ctx context.Context, token string, singleUse bool) (*providers.ExternalIdentity, error) {
|
||||
if !p.initialized {
|
||||
return nil, fmt.Errorf("provider not initialized")
|
||||
}
|
||||
@@ -551,7 +491,7 @@ func (p *OIDCProvider) authenticate(ctx context.Context, token string, singleUse
|
||||
}
|
||||
|
||||
// Validate token and get claims
|
||||
claims, err := p.validateToken(ctx, token, singleUse)
|
||||
claims, err := p.ValidateToken(ctx, token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -718,16 +658,6 @@ func (p *OIDCProvider) getUserInfoWithToken(ctx context.Context, userID, accessT
|
||||
|
||||
// ValidateToken validates an OIDC JWT token
|
||||
func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
|
||||
return p.validateToken(ctx, token, false)
|
||||
}
|
||||
|
||||
// ValidateTokenSingleUse validates and consumes the token: a second
|
||||
// presentation of it is rejected as a replay. See AuthenticateSingleUse.
|
||||
func (p *OIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
|
||||
return p.validateToken(ctx, token, true)
|
||||
}
|
||||
|
||||
func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUse bool) (*providers.TokenClaims, error) {
|
||||
if !p.initialized {
|
||||
return nil, fmt.Errorf("provider not initialized")
|
||||
}
|
||||
@@ -754,7 +684,8 @@ func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUs
|
||||
return nil, fmt.Errorf("failed to get public key: %v", err)
|
||||
}
|
||||
|
||||
// Parse and validate token with proper signature verification
|
||||
// Parse and validate token with proper signature verification. exp is
|
||||
// required: without it a leaked token could mint STS sessions forever.
|
||||
claims := jwt.MapClaims{}
|
||||
validatedToken, err := jwt.ParseWithClaims(token, claims, func(token *jwt.Token) (interface{}, error) {
|
||||
// Verify signing method
|
||||
@@ -764,7 +695,7 @@ func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUs
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported signing method: %v", token.Header["alg"])
|
||||
}
|
||||
})
|
||||
}, jwt.WithExpirationRequired())
|
||||
|
||||
if err != nil {
|
||||
// Use JWT library's typed errors for robust error checking
|
||||
@@ -860,16 +791,6 @@ func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUs
|
||||
tokenClaims.Claims[key] = value
|
||||
}
|
||||
|
||||
// Consume the token for replay protection. The check runs only for
|
||||
// single-use callers: a bearer token is meant to be presented on every
|
||||
// request, so validating it must not consume it.
|
||||
if singleUse {
|
||||
jti, _ := claims["jti"].(string)
|
||||
if err := p.replayCache().consume(token, jti, subject, tokenClaims.ExpiresAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return tokenClaims, nil
|
||||
}
|
||||
|
||||
@@ -1298,36 +1219,3 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p
|
||||
|
||||
return identity
|
||||
}
|
||||
|
||||
// Shutdown gracefully stops the JTI cleanup goroutine.
|
||||
// Should be called when the provider is being shut down to prevent goroutine leaks.
|
||||
func (p *OIDCProvider) Shutdown(ctx context.Context) error {
|
||||
return p.replayCache().stop(ctx)
|
||||
}
|
||||
|
||||
// AdoptReplayStateFrom takes over the consumed-token store of the provider it
|
||||
// replaces, so tokens it already saw stay rejected across the rebuild. The
|
||||
// caller must shut down the replaced provider; only one cleanup worker may
|
||||
// sweep the shared store at a time. Call before the new provider serves
|
||||
// validators.
|
||||
func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) {
|
||||
if old == nil || old == p || old.jti == nil {
|
||||
return
|
||||
}
|
||||
p.replayCache().adoptFrom(old.jti)
|
||||
}
|
||||
|
||||
// MergeReplayStateFrom hands this provider's store to another provider and
|
||||
// then folds the other's consumed-token entries in, so rejections the other
|
||||
// recorded keep holding when its records move to a shared provider. Repointing
|
||||
// first means tokens the other still consumes — in-flight exchanges or ones
|
||||
// routed to it before the provider map updates — land in the shared store
|
||||
// instead of a store this merge already drained. Unlike AdoptReplayStateFrom
|
||||
// it swaps nothing of its own, so it is safe on a provider that is already
|
||||
// serving validators.
|
||||
func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) {
|
||||
if other == nil || other == p {
|
||||
return
|
||||
}
|
||||
p.replayCache().mergeFrom(other.replayCache())
|
||||
}
|
||||
+24
-1080
File diff suppressed because it is too large.
Load diff
@@ -19,7 +19,4 @@ var (
|
||||
|
||||
// ErrProviderMissingClaims indicates that required claims are missing from the token
|
||||
ErrProviderMissingClaims = errors.New("provider: missing required claims")
|
||||
|
||||
// ErrProviderTokenReplayed is returned when a token with duplicate JTI is detected
|
||||
ErrProviderTokenReplayed = errors.New("provider: token has been replayed")
|
||||
)
|
||||
+15
-17
@@ -42,23 +42,21 @@ const (
|
||||
|
||||
// Configuration Field Names
|
||||
const (
|
||||
ConfigFieldFilerAddress = "filerAddress"
|
||||
ConfigFieldBasePath = "basePath"
|
||||
ConfigFieldIssuer = "issuer"
|
||||
ConfigFieldClientID = "clientId"
|
||||
ConfigFieldClientIDs = "clientIds"
|
||||
ConfigFieldThumbprints = "thumbprints"
|
||||
ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys"
|
||||
ConfigFieldPolicyClaim = "policyClaim"
|
||||
ConfigFieldClientSecret = "clientSecret"
|
||||
ConfigFieldJWKSUri = "jwksUri"
|
||||
ConfigFieldScopes = "scopes"
|
||||
ConfigFieldUserInfoUri = "userInfoUri"
|
||||
ConfigFieldRedirectUri = "redirectUri"
|
||||
ConfigFieldTLSCACert = "tlsCaCert"
|
||||
ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify"
|
||||
ConfigFieldJTIReplayProtectionEnabled = "jtiReplayProtectionEnabled"
|
||||
ConfigFieldJTICacheMaxSize = "jtiCacheMaxSize"
|
||||
ConfigFieldFilerAddress = "filerAddress"
|
||||
ConfigFieldBasePath = "basePath"
|
||||
ConfigFieldIssuer = "issuer"
|
||||
ConfigFieldClientID = "clientId"
|
||||
ConfigFieldClientIDs = "clientIds"
|
||||
ConfigFieldThumbprints = "thumbprints"
|
||||
ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys"
|
||||
ConfigFieldPolicyClaim = "policyClaim"
|
||||
ConfigFieldClientSecret = "clientSecret"
|
||||
ConfigFieldJWKSUri = "jwksUri"
|
||||
ConfigFieldScopes = "scopes"
|
||||
ConfigFieldUserInfoUri = "userInfoUri"
|
||||
ConfigFieldRedirectUri = "redirectUri"
|
||||
ConfigFieldTLSCACert = "tlsCaCert"
|
||||
ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify"
|
||||
)
|
||||
|
||||
// Error Messages
|
||||
|
||||
@@ -103,51 +103,3 @@ func TestLookupOIDCProviderForAccountFallsBackToStatic(t *testing.T) {
|
||||
t.Fatalf("expected static-config provider as last resort, got %v ok=%v", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// shutdownTrackingProvider records Shutdown calls so tests can observe which
|
||||
// replaced providers the swap released.
|
||||
type shutdownTrackingProvider struct {
|
||||
stubIdentityProvider
|
||||
shutdownCalls int32
|
||||
}
|
||||
|
||||
func (s *shutdownTrackingProvider) Shutdown(context.Context) error {
|
||||
s.shutdownCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
// A provider dropped from the managed map owns a background JTI cleanup
|
||||
// worker; replacing the map must shut it down, while providers that stay in
|
||||
// the map keep running.
|
||||
func TestSetIAMManagedOIDCProvidersShutsDownDroppedProviders(t *testing.T) {
|
||||
dropped := &shutdownTrackingProvider{stubIdentityProvider{name: "dropped"}, 0}
|
||||
kept := &shutdownTrackingProvider{stubIdentityProvider{name: "kept"}, 0}
|
||||
added := &shutdownTrackingProvider{stubIdentityProvider{name: "added"}, 0}
|
||||
|
||||
s := &STSService{issuerToProvider: map[string]providers.IdentityProvider{}}
|
||||
s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
|
||||
"https://a.example.com": {{Provider: dropped}},
|
||||
"https://b.example.com": {{Provider: kept}, {AccountID: "111111111111", Provider: kept}},
|
||||
})
|
||||
|
||||
s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
|
||||
"https://b.example.com": {{Provider: kept}},
|
||||
"https://c.example.com": {{Provider: added}},
|
||||
})
|
||||
|
||||
if dropped.shutdownCalls != 1 {
|
||||
t.Fatalf("dropped provider was shut down %d times, want 1", dropped.shutdownCalls)
|
||||
}
|
||||
if kept.shutdownCalls != 0 {
|
||||
t.Fatalf("kept provider was shut down %d times, want 0", kept.shutdownCalls)
|
||||
}
|
||||
if added.shutdownCalls != 0 {
|
||||
t.Fatalf("newly added provider was shut down %d times, want 0", added.shutdownCalls)
|
||||
}
|
||||
|
||||
// Clearing the map releases the remaining providers.
|
||||
s.SetIAMManagedOIDCProviders(nil)
|
||||
if kept.shutdownCalls != 1 || added.shutdownCalls != 1 {
|
||||
t.Fatalf("clearing the map shut down kept=%d added=%d, want 1 each", kept.shutdownCalls, added.shutdownCalls)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package sts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -155,21 +154,6 @@ func (f *ProviderFactory) convertToOIDCConfig(configMap map[string]interface{})
|
||||
config.TLSInsecureSkipVerify = tlsInsecureSkipVerify
|
||||
}
|
||||
|
||||
if enabled, ok := configMap[ConfigFieldJTIReplayProtectionEnabled].(bool); ok {
|
||||
config.JTIReplayProtectionEnabled = &enabled
|
||||
}
|
||||
|
||||
switch maxSize := configMap[ConfigFieldJTICacheMaxSize].(type) {
|
||||
case float64:
|
||||
config.JTICacheMaxSize = int(maxSize)
|
||||
case int:
|
||||
config.JTICacheMaxSize = maxSize
|
||||
case json.Number:
|
||||
if n, err := maxSize.Int64(); err == nil {
|
||||
config.JTICacheMaxSize = int(n)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert claims mapping
|
||||
if claimsMapInterface, ok := configMap["claimsMapping"]; ok {
|
||||
claimsMap, err := f.convertToStringMap(claimsMapInterface)
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package sts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConvertToOIDCConfigCarriesJTISettings(t *testing.T) {
|
||||
factory := NewProviderFactory()
|
||||
|
||||
config, err := factory.convertToOIDCConfig(map[string]interface{}{
|
||||
"issuer": "https://issuer.example",
|
||||
"clientId": "client",
|
||||
"jtiReplayProtectionEnabled": false,
|
||||
"jtiCacheMaxSize": float64(42),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, config.JTIReplayProtectionEnabled)
|
||||
assert.False(t, *config.JTIReplayProtectionEnabled)
|
||||
assert.Equal(t, 42, config.JTICacheMaxSize)
|
||||
|
||||
config, err = factory.convertToOIDCConfig(map[string]interface{}{
|
||||
"issuer": "https://issuer.example",
|
||||
"clientId": "client",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, config.JTIReplayProtectionEnabled)
|
||||
assert.Zero(t, config.JTICacheMaxSize)
|
||||
}
|
||||
@@ -522,35 +522,8 @@ func (s *STSService) SetIAMManagedOIDCProviders(byIssuer map[string][]ScopedOIDC
|
||||
}
|
||||
}
|
||||
s.iamManagedOIDCMu.Lock()
|
||||
old := s.iamManagedOIDCByIssuer
|
||||
s.iamManagedOIDCByIssuer = cp
|
||||
s.iamManagedOIDCMu.Unlock()
|
||||
|
||||
// Providers dropped by the swap may own background workers (the JTI
|
||||
// cleanup goroutine); stop them so a replaced provider does not leak.
|
||||
kept := make(map[providers.IdentityProvider]struct{})
|
||||
for _, scoped := range cp {
|
||||
for _, sp := range scoped {
|
||||
kept[sp.Provider] = struct{}{}
|
||||
}
|
||||
}
|
||||
stopped := make(map[providers.IdentityProvider]struct{})
|
||||
for _, scoped := range old {
|
||||
for _, sp := range scoped {
|
||||
if _, ok := kept[sp.Provider]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := stopped[sp.Provider]; ok {
|
||||
continue
|
||||
}
|
||||
stopped[sp.Provider] = struct{}{}
|
||||
if sd, ok := sp.Provider.(interface{ Shutdown(context.Context) error }); ok {
|
||||
if err := sd.Shutdown(context.Background()); err != nil {
|
||||
glog.Warningf("OIDC provider %q shutdown after replacement: %v", sp.Provider.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookupOIDCProviderForAccount returns the provider that should validate
|
||||
@@ -624,7 +597,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
|
||||
// caller passed a legacy or claim-based ARN, in which case lookup falls
|
||||
// back to a global (account-less) record only.
|
||||
roleAccountID := utils.ParseRoleARN(request.RoleArn).AccountID
|
||||
externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID, true)
|
||||
externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to validate web identity token: %w", err)
|
||||
}
|
||||
@@ -1029,7 +1002,7 @@ func (s *STSService) validateAssumeRoleWithWebIdentityRequest(request *AssumeRol
|
||||
// validateWebIdentityToken validates the web identity token with strict issuer-to-provider mapping
|
||||
// SECURITY: JWT tokens with a specific issuer claim MUST only be validated by the provider for that issuer
|
||||
// SECURITY: This method only accepts JWT tokens. Non-JWT authentication must use AssumeRoleWithCredentials with explicit ProviderName.
|
||||
func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string, singleUse bool) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
|
||||
func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
|
||||
// Try to extract issuer from JWT token for strict validation
|
||||
issuer, err := s.extractIssuerFromJWT(token)
|
||||
if err != nil {
|
||||
@@ -1052,10 +1025,8 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
|
||||
return nil, nil, fmt.Errorf("no identity provider registered for issuer: %s", issuer)
|
||||
}
|
||||
|
||||
// Authenticate with the correct provider for this issuer. Only the STS
|
||||
// exchange authenticates single-use: a bearer token is presented on every
|
||||
// request, so consuming it here would reject all but the first one.
|
||||
identity, err := authenticateOIDCProvider(ctx, provider, token, singleUse)
|
||||
// Authenticate with the correct provider for this issuer
|
||||
identity, err := provider.Authenticate(ctx, token)
|
||||
if err != nil {
|
||||
// Map provider errors to STS errors using errors.Is() for robust error checking
|
||||
// This eliminates fragile string matching and provides reliable error classification
|
||||
@@ -1087,29 +1058,14 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
|
||||
// only, then static-config); call ValidateWebIdentityTokenForAccount when the assumed-role account
|
||||
// is known.
|
||||
func (s *STSService) ValidateWebIdentityToken(ctx context.Context, token string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
|
||||
return s.validateWebIdentityToken(ctx, token, "", false)
|
||||
return s.validateWebIdentityToken(ctx, token, "")
|
||||
}
|
||||
|
||||
// ValidateWebIdentityTokenForAccount mirrors ValidateWebIdentityToken but
|
||||
// scopes the IAM-managed provider lookup to roleAccountID. Pass "" for
|
||||
// callers that don't yet know the account (e.g. claim-based mode).
|
||||
func (s *STSService) ValidateWebIdentityTokenForAccount(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
|
||||
return s.validateWebIdentityToken(ctx, token, roleAccountID, false)
|
||||
}
|
||||
|
||||
// authenticateOIDCProvider authenticates the token, consuming it for replay
|
||||
// protection when the caller is the single-use STS exchange and the provider
|
||||
// supports it. Other providers and non-exchange callers authenticate without
|
||||
// consuming.
|
||||
func authenticateOIDCProvider(ctx context.Context, provider providers.IdentityProvider, token string, singleUse bool) (*providers.ExternalIdentity, error) {
|
||||
if singleUse {
|
||||
if su, ok := provider.(interface {
|
||||
AuthenticateSingleUse(context.Context, string) (*providers.ExternalIdentity, error)
|
||||
}); ok {
|
||||
return su.AuthenticateSingleUse(ctx, token)
|
||||
}
|
||||
}
|
||||
return provider.Authenticate(ctx, token)
|
||||
return s.validateWebIdentityToken(ctx, token, roleAccountID)
|
||||
}
|
||||
|
||||
// extractIssuerFromJWT extracts the issuer (iss) claim from a JWT token without verification
|
||||
|
||||
Loaded 100 of 140 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user