diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml new file mode 100644 index 0000000000..fc93694466 --- /dev/null +++ b/.github/workflows/posix-compliance.yml @@ -0,0 +1,777 @@ +name: POSIX Compliance Tests + +on: + push: + branches: [ main, master, develop ] + paths: + - 'weed/mount/**' + - 'weed/command/mount*' + - 'weed/command/fuse*' + - 'test/fuse_integration/**' + pull_request: + branches: [ main, master, develop ] + paths: + - 'weed/mount/**' + - 'weed/command/mount*' + - 'weed/command/fuse*' + - 'test/fuse_integration/**' + workflow_dispatch: + inputs: + test_type: + description: 'Type of POSIX tests to run (default: full for comprehensive testing)' + required: true + default: 'full' + type: choice + options: + - critical + - basic + - extended + - full + enable_external_tests: + description: 'Run external test suites (slower)' + required: false + default: false + type: boolean + +permissions: + contents: read + pull-requests: write + issues: write + +env: + GO_VERSION: '1.26' + TIMEOUT: '45m' + +jobs: + posix-compliance-ubuntu: + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + matrix: + fuse-version: ['2.9', '3.0'] + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + fetch-depth: 1 + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: | + go.sum + test/fuse_integration/go.sum + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + attr \ + acl \ + build-essential \ + git \ + python3-pip + + # Install FUSE version specific packages + if [ "${{ matrix.fuse-version }}" = "2.9" ]; then + echo "Installing FUSE 2.9..." + sudo apt-get install -y fuse libfuse-dev + elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then + echo "Installing FUSE 3.0..." + sudo apt-get install -y fuse3 libfuse3-dev + # Also install fuse2 for compatibility if needed + sudo apt-get install -y fuse libfuse-dev + fi + + - name: Set up user permissions for FUSE + run: | + # Create fuse group if it doesn't exist + sudo groupadd -f fuse + + # Add user to fuse group + sudo usermod -a -G fuse $USER + + # Set permissions on FUSE device + sudo chmod 666 /dev/fuse + + # Ensure fuse module is loaded + sudo modprobe fuse || true + + # Verify setup + echo "FUSE setup verification:" + ls -la /dev/fuse + groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)" + lsmod | grep fuse || echo "FUSE module not loaded" + + - name: Install external test tools + if: ${{ github.event.inputs.enable_external_tests == 'true' }} + run: | + # Install nfstest for POSIX API verification + pip3 install --user nfstest + + # Install FIO for performance testing + sudo apt-get install -y fio + + # Install additional test utilities + sudo apt-get install -y stress-ng + + - name: Build SeaweedFS + run: | + make + # Verify binary exists and is executable (it's installed to GOPATH/bin) + weed version + + # Make weed binary available in PATH (it's already there from go install) + sudo cp $(which weed) /usr/local/bin/weed + which weed + weed version + + - name: Set up SeaweedFS cluster + run: | + start_and_wait() { + local name=$1 + local url=$2 + local pidfile=$3 + shift 3 + local cmd="$@" + + echo "Starting $name..." + $cmd > "/tmp/seaweedfs/$name.log" 2>&1 & + local pid=$! + echo $pid > "$pidfile" + + echo "Waiting for $name to start..." + for i in {1..30}; do + if curl -sf "$url" > /dev/null 2>&1; then + echo "$name is ready" + return 0 + fi + if [ $i -eq 30 ]; then + echo "$name failed to start" + cat "/tmp/seaweedfs/$name.log" + exit 1 + fi + sleep 2 + done + } + + mkdir -p /tmp/seaweedfs/{master,volume,filer,mount} + + start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \ + "weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true" + + start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \ + "weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100" + + start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \ + "weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888" + + echo "SeaweedFS cluster status:" + curl -s http://127.0.0.1:9333/cluster/status || true + + - name: Set up FUSE mount + run: | + # Create mount point + MOUNT_POINT="/tmp/seaweedfs/mount" + mkdir -p $MOUNT_POINT + + echo "Mounting SeaweedFS FUSE filesystem..." + # Mount SeaweedFS FUSE filesystem in background + weed mount \ + -filer=127.0.0.1:8888 \ + -dir=$MOUNT_POINT \ + -filer.path=/ \ + -dirAutoCreate=true \ + -allowOthers=true \ + -nonempty=true \ + > /tmp/seaweedfs/mount.log 2>&1 & + MOUNT_PID=$! + echo $MOUNT_PID > /tmp/seaweedfs/mount.pid + + # Wait for mount to be ready. The mount point directory already + # exists, so only a real mount counts — a directory listing would + # pass on the local disk even if weed mount died. + echo "Waiting for FUSE mount to be ready..." + for i in {1..30}; do + if mountpoint -q $MOUNT_POINT 2>/dev/null; then + echo "FUSE mount is ready" + break + fi + if ! kill -0 $MOUNT_PID 2>/dev/null; then + echo "FUSE mount process exited" + cat /tmp/seaweedfs/mount.log + exit 1 + fi + if [ $i -eq 30 ]; then + echo "FUSE mount failed to be ready" + cat /tmp/seaweedfs/mount.log + echo "Mount status:" + mount | grep fuse || echo "No FUSE mounts found" + ps aux | grep weed || echo "No weed processes found" + exit 1 + fi + sleep 2 + done + + # Verify mount is working + echo "Testing FUSE mount functionality..." + echo "test" > $MOUNT_POINT/test_file.txt + if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then + echo "FUSE mount is working correctly" + rm $MOUNT_POINT/test_file.txt + else + echo "FUSE mount is not working properly" + exit 1 + fi + + # Export mount point for tests + echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV + + - name: Set up test environment + run: | + cd test/fuse_integration + + # Initialize Go module for tests + if [ ! -f go.mod ]; then + go mod init seaweedfs-posix-tests + go mod tidy + fi + + # Create reports directory + mkdir -p reports + + # Set up external tools if requested + # Note: External tools setup is now optional and won't fail the build + if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then + echo "Setting up external POSIX testing tools..." + make -f posix_Makefile setup-external-tools || true + # setup-external-tools exits 0 even when a sub-target failed, so + # require the tools only when they are actually installed + if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then + export POSIX_REQUIRE_EXTERNAL_TOOLS="true" + echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV + echo "✅ External tools setup completed successfully" + else + echo "⚠️ External tools setup incomplete - continuing with basic tests only" + echo "External tools will be skipped but tests will continue" + echo "This is expected behavior when build tools are not available" + fi + fi + + # Verify SeaweedFS cluster is accessible + echo "Verifying SeaweedFS cluster accessibility..." + curl -sf http://127.0.0.1:9333/cluster/status + curl -sf http://127.0.0.1:8080/status + curl -sf http://127.0.0.1:8888/ + + # Verify mount point + echo "Mount point: $SEAWEEDFS_MOUNT_POINT" + ls -la $SEAWEEDFS_MOUNT_POINT + + # Verify mount is actually working by testing file operations + echo "Verifying SeaweedFS mount functionality..." + TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test" + echo "SeaweedFS mount test" > "$TEST_FILE" + if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then + echo "✅ Mount verification successful" + rm -f "$TEST_FILE" + else + echo "❌ Mount verification failed - mount may not be working properly" + exit 1 + fi + + # Check if this is actually a FUSE mount — the plain directory created + # above passes the file test, so without this the suite can run + # against the local disk and report a pass it never earned + if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then + echo "✅ FUSE mount detected:" + mount | grep "$SEAWEEDFS_MOUNT_POINT" + else + echo "❌ Mount point not found in mount table - refusing to test a local directory" + exit 1 + fi + + - name: Run POSIX compliance tests + id: posix-tests + env: + SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }} + POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }} + run: | + cd test/fuse_integration + + # Determine which tests to run + TEST_TYPE="${{ github.event.inputs.test_type }}" + if [ -z "$TEST_TYPE" ]; then + TEST_TYPE="full" + fi + + echo "Running POSIX tests: $TEST_TYPE" + if [ "$TEST_TYPE" = "full" ]; then + echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)" + else + echo "Using mount point: $SEAWEEDFS_MOUNT_POINT" + fi + + # Set test configuration to use our mounted filesystem + export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT" + export TEST_SKIP_CLUSTER_SETUP="true" + + # Debug: Show environment variables + echo "🔍 Test Environment:" + echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT" + echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP" + echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT" + + case "$TEST_TYPE" in + "critical") + echo "Running critical POSIX tests (basic compliance only)..." + make -f posix_Makefile test-posix-critical + ;; + "basic") + echo "Running basic POSIX tests (core file operations)..." + make -f posix_Makefile test-posix-basic + ;; + "extended") + echo "Running extended POSIX tests (including advanced features)..." + make -f posix_Makefile test-posix-extended + ;; + "full") + echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..." + echo " This includes: basic, extended, and external test suites" + echo " External suites: pjdfstest, nfstest_posix, and custom stress tests" + make -f posix_Makefile test-posix-full + ;; + *) + echo "Unknown test type: $TEST_TYPE, defaulting to full suite" + echo "🚀 Running full POSIX test suite (all tests including external)..." + make -f posix_Makefile test-posix-full + ;; + esac + + - name: Run external test suites + if: ${{ github.event.inputs.enable_external_tests == 'true' }} + continue-on-error: true + run: | + cd test/fuse_integration + + # Point the external suites at the same mount as the Go tests; + # without these the Makefile falls back to a local-disk path. + export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT" + export TEST_SKIP_CLUSTER_SETUP="true" + + # Run external tests (may fail on some systems) + make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available" + make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available" + + - name: Generate compliance report + if: always() + run: | + cd test/fuse_integration + make -f posix_Makefile generate-report + + - name: Collect system information for debugging + if: failure() + run: | + cd test/fuse_integration + mkdir -p reports/debug + + echo "System Information" > reports/debug/system_info.txt + uname -a >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "Mount Information" >> reports/debug/system_info.txt + mount >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "Disk Space" >> reports/debug/system_info.txt + df -h >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "Memory Information" >> reports/debug/system_info.txt + free -h >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "FUSE Information" >> reports/debug/system_info.txt + ls -la /dev/fuse >> reports/debug/system_info.txt + lsmod | grep fuse >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "SeaweedFS Version" >> reports/debug/system_info.txt + weed version >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "Go Version" >> reports/debug/system_info.txt + go version >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + echo "Running Processes" >> reports/debug/system_info.txt + ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + + # Collect SeaweedFS service logs + echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt + if [ -f /tmp/seaweedfs/master.log ]; then + echo "Master Log:" >> reports/debug/system_info.txt + tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + fi + + if [ -f /tmp/seaweedfs/volume.log ]; then + echo "Volume Log:" >> reports/debug/system_info.txt + tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + fi + + if [ -f /tmp/seaweedfs/filer.log ]; then + echo "Filer Log:" >> reports/debug/system_info.txt + tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + fi + + if [ -f /tmp/seaweedfs/mount.log ]; then + echo "Mount Log:" >> reports/debug/system_info.txt + tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt + echo "" >> reports/debug/system_info.txt + fi + + # Copy full logs as separate files + if [ -f /tmp/seaweedfs/master.log ]; then + cp /tmp/seaweedfs/master.log reports/debug/master.log + fi + if [ -f /tmp/seaweedfs/volume.log ]; then + cp /tmp/seaweedfs/volume.log reports/debug/volume.log + fi + if [ -f /tmp/seaweedfs/filer.log ]; then + cp /tmp/seaweedfs/filer.log reports/debug/filer.log + fi + if [ -f /tmp/seaweedfs/mount.log ]; then + cp /tmp/seaweedfs/mount.log reports/debug/mount.log + fi + + - name: Cleanup SeaweedFS cluster and FUSE mount + if: always() + run: | + stop_process() { + local name=$1 + local pidfile=$2 + if [ -f "$pidfile" ]; then + local pid=$(cat "$pidfile") + if kill -0 $pid 2>/dev/null; then + echo "Stopping $name process (PID: $pid)..." + kill -TERM $pid || true + sleep 2 + kill -KILL $pid 2>/dev/null || true + fi + fi + } + + echo "Cleaning up SeaweedFS cluster and FUSE mount..." + + MOUNT_POINT="/tmp/seaweedfs/mount" + if mountpoint -q $MOUNT_POINT 2>/dev/null; then + echo "Unmounting FUSE filesystem..." + fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true + fi + + stop_process "mount" "/tmp/seaweedfs/mount.pid" + stop_process "filer" "/tmp/seaweedfs/filer.pid" + stop_process "volume" "/tmp/seaweedfs/volume.pid" + stop_process "master" "/tmp/seaweedfs/master.pid" + + pkill -f "weed " || true + + fusermount -u $MOUNT_POINT 2>/dev/null || true + umount $MOUNT_POINT 2>/dev/null || true + + rm -rf /tmp/seaweedfs || true + + echo "Cleanup completed" + + + - name: Upload test results + uses: actions/upload-artifact@v7 + if: always() + with: + name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }} + path: | + test/fuse_integration/reports/ + test/fuse_integration/*.log + retention-days: 30 + + - name: Upload test coverage + uses: actions/upload-artifact@v7 + if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended') + with: + name: posix-coverage-ubuntu + path: test/fuse_integration/reports/posix_coverage.html + retention-days: 14 + + - name: Comment PR with results + if: github.event_name == 'pull_request' && always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt'; + + if (fs.existsSync(reportPath)) { + const report = fs.readFileSync(reportPath, 'utf8'); + const comment = `## POSIX Compliance Test Results + + **Go Version:** ${{ env.GO_VERSION }} + **FUSE Version:** ${{ matrix.fuse-version }} + **Test Type:** ${{ github.event.inputs.test_type || 'full' }} + +
+ Test Summary + + \`\`\` + ${report} + \`\`\` +
+ + 📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) + `; + + try { + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } catch (err) { + console.log('Could not post PR comment: ' + err.message); + } + } + + posix-compliance-macos: + runs-on: macos-latest + timeout-minutes: 60 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Install macFUSE + run: | + # Install macFUSE + brew install macfuse + + # Note: macFUSE may require system extension approval on macOS + # This step may need manual intervention in some cases + + - name: Build SeaweedFS + run: | + make + # Verify binary exists and is executable (it's installed to GOPATH/bin) + weed version + + - name: Run critical POSIX tests (macOS) + continue-on-error: true # macOS FUSE can be more restrictive + run: | + cd test/fuse_integration + + if [ ! -f go.mod ]; then + go mod init seaweedfs-posix-tests + go mod tidy + fi + + mkdir -p reports + + # Run basic tests only on macOS due to FUSE limitations + make -f posix_Makefile test-posix-critical + + - name: Upload macOS test results + uses: actions/upload-artifact@v7 + if: always() + with: + name: posix-test-results-macos + path: | + test/fuse_integration/reports/ + test/fuse_integration/*.log + retention-days: 30 + + security-analysis: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install security analysis tools + run: | + echo "Installing staticcheck for security analysis..." + go install honnef.co/go/tools/cmd/staticcheck@latest + + - name: Run security analysis on FUSE code + run: | + # Analyze mount and FUSE-related code for security and quality issues + echo "Running staticcheck security analysis..." + + # Run staticcheck on FUSE-related code + echo "Analyzing FUSE mount code..." + staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true + staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true + + # Convert to JSON format for consistency + echo "{" > gosec-report.json + echo " \"tool\": \"staticcheck\"," >> gosec-report.json + echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json + echo " \"issues\": [" >> gosec-report.json + + if [ -s staticcheck-report.txt ]; then + echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json + fi + + echo " ]," >> gosec-report.json + echo " \"stats\": {" >> gosec-report.json + echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json + echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json + echo " }" >> gosec-report.json + echo "}" >> gosec-report.json + + echo "Security analysis completed" + echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" + + - name: Upload security analysis results + uses: actions/upload-artifact@v7 + if: always() + with: + name: security-analysis-results + path: gosec-report.json + retention-days: 30 + + performance-baseline: + runs-on: ubuntu-latest + if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule' + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y fuse libfuse-dev fio + sudo usermod -a -G fuse $USER + sudo chmod 666 /dev/fuse + + - name: Build SeaweedFS + run: make + + - name: Run performance baseline tests + run: | + cd test/fuse_integration + + if [ ! -f go.mod ]; then + go mod init seaweedfs-posix-tests + go mod tidy + fi + + mkdir -p reports + + # Run performance benchmarks + make -f posix_Makefile benchmark-posix + # This job has no FUSE mount, so the fio suite can only skip; keep the + # run green but honest about producing no mounted-FS results + make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job" + + - name: Store performance baseline + uses: actions/upload-artifact@v7 + with: + name: performance-baseline-results + path: | + test/fuse_integration/reports/posix_benchmark_results.log + test/fuse_integration/reports/fio_*_results.log + retention-days: 90 + + # Schedule regular compliance checks + scheduled-compliance-check: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y fuse libfuse-dev + pip3 install --user nfstest + + - name: Build SeaweedFS + run: make + + - name: Run comprehensive compliance check + run: | + cd test/fuse_integration + make -f posix_Makefile test-posix-full + make -f posix_Makefile generate-report + + - name: Create compliance issue if tests fail + if: failure() + uses: actions/github-script@v7 + with: + script: | + const issue = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0], + body: `## POSIX Compliance Check Failure + + The scheduled POSIX compliance check has failed. This may indicate: + + - Regression in FUSE mount functionality + - Changes affecting POSIX compatibility + - Infrastructure issues with the test environment + + **Action Required:** Review the test results and investigate any failures. + + **Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) + + **Date:** ${new Date().toISOString()} + + --- + This issue was automatically created by the POSIX compliance workflow. + `, + labels: ['bug', 'posix-compliance', 'automated-issue'] + }); + + console.log('Created issue:', issue.data.number); + +# Schedule this workflow to run weekly +# Uncomment the following lines to enable scheduled runs: +# schedule: +# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC + diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 4935970297..247b5bd06c 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "4.48" +appVersion: "4.49" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.48.2 +version: 4.49.0 diff --git a/note/star_history.svg b/note/star_history.svg index 0f1ef8ea20..ad8995f239 100644 --- a/note/star_history.svg +++ b/note/star_history.svg @@ -6,7 +6,7 @@ - 2026-10-21T00:43:43.329650 + 2026-10-23T02:28:24.716056 image/svg+xml @@ -40,23 +40,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -301,18 +301,18 @@ L 353.549098 26.215385 - + - + - + @@ -325,18 +325,18 @@ L 449.518085 26.215385 - + - + - + @@ -349,18 +349,18 @@ L 545.355787 26.215385 - + - + - + @@ -484,41 +484,41 @@ z - + - - + - + - + - + - + - + - + - + @@ -577,18 +577,18 @@ L 709.2 267.624779 - + - + - + @@ -599,18 +599,18 @@ L 709.2 223.613973 - + - + - + @@ -621,18 +621,18 @@ L 709.2 179.603167 - + - + - + @@ -643,18 +643,18 @@ L 709.2 135.592361 - + - + - + - + - + - + @@ -813,1028 +813,1023 @@ z +" clip-path="url(#pfa329b79af)" style="fill: none; stroke: #0969da; stroke-width: 1.6; stroke-linecap: square"/> + diff --git a/seaweed-volume/Cargo.lock b/seaweed-volume/Cargo.lock index 92f4c3471a..afabe1c8aa 100644 --- a/seaweed-volume/Cargo.lock +++ b/seaweed-volume/Cargo.lock @@ -86,7 +86,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -510,10 +510,12 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.5" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] @@ -563,11 +565,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.6" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -579,10 +582,32 @@ dependencies = [ ] [[package]] -name = "aws-smithy-types" -version = "1.4.7" +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.0", +] + +[[package]] +name = "aws-smithy-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" dependencies = [ "base64-simd", "bytes", @@ -1325,7 +1350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2379,7 +2404,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3391,7 +3416,7 @@ dependencies = [ "errno 0.3.14", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3676,7 +3701,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno 0.3.14", + "errno 0.2.8", "libc", ] @@ -3743,7 +3768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3877,7 +3902,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index e1e12fb608..4cef145590 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{ }; use crate::storage::needle::needle::{Needle, get_actual_size}; use crate::storage::needle_map::NeedleMapKind; -use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE}; +use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock}; use crate::storage::types::*; use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec}; @@ -762,8 +762,8 @@ impl Store { } /// Configure a volume's replica placement on disk. - /// The volume must already be unmounted. This opens the .dat file directly, - /// modifies the replica_placement byte (offset 1), and writes it back. + /// The volume must already be unmounted. Rewrites the .vif and the .dat + /// superblock so both carry the new placement. pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> { let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| { VolumeError::Io(io::Error::new( @@ -775,7 +775,7 @@ impl Store { let mut vif = load_vif_volume_info(&vif_path)?; vif.replication = rp.to_string(); save_vif_volume_info(&vif_path, &vif)?; - Ok(()) + update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp) } // ---- Read / Write / Delete ---- @@ -1740,6 +1740,61 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> { .or_else(|| filename.strip_suffix(".idx")) } +/// Rewrites the replica placement in a .dat header for a volume that is not +/// mounted; a missing .dat means a remote-only volume, where the .vif carries +/// the placement. +fn update_dat_replica_placement( + dat_path: &str, + vid: VolumeId, + rp: ReplicaPlacement, +) -> Result<(), VolumeError> { + use std::io::{Read, Seek, SeekFrom, Write}; + + let mut file = match std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(dat_path) + { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(VolumeError::Io(io::Error::new( + e.kind(), + format!("volume {} failed to open {}: {}", vid, dat_path, e), + ))) + } + }; + let mut header = [0u8; SUPER_BLOCK_SIZE]; + file.read_exact(&mut header).map_err(|e| { + VolumeError::Io(io::Error::new( + e.kind(), + format!("volume {} failed to parse superblock: {}", vid, e), + )) + })?; + let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize; + let mut block = header.to_vec(); + block.resize(SUPER_BLOCK_SIZE + extra_size, 0); + if extra_size > 0 { + file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| { + VolumeError::Io(io::Error::new( + e.kind(), + format!("volume {} failed to parse superblock: {}", vid, e), + )) + })?; + } + let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| { + VolumeError::Io(io::Error::new( + io::ErrorKind::InvalidData, + format!("volume {} failed to parse superblock: {}", vid, e), + )) + })?; + super_block.replica_placement = rp; + file.seek(SeekFrom::Start(0))?; + file.write_all(&super_block.to_bytes())?; + file.sync_all()?; + Ok(()) +} + fn load_vif_volume_info(path: &str) -> Result { let content = match std::fs::read_to_string(path) { Ok(c) => c, @@ -1977,6 +2032,33 @@ mod tests { assert_eq!(store.total_volume_count(), 1); } + // Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave + // the .dat header and the .vif both on the new placement. + #[test] + fn test_configure_volume_updates_superblock() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let mut store = make_test_store(&[dir]); + + store + .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default()) + .unwrap(); + assert!(store.unmount_volume(VolumeId(1)).unwrap()); + + let rp = ReplicaPlacement::from_string("010").unwrap(); + store.configure_volume(VolumeId(1), rp).unwrap(); + + let base = volume_file_name(dir, "", VolumeId(1)); + let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap(); + assert_eq!(vif.replication, "010"); + + let dat = std::fs::read(format!("{}.dat", base)).unwrap(); + let super_block = SuperBlock::from_bytes(&dat).unwrap(); + assert_eq!(super_block.replica_placement, rp); + + assert!(store.configure_volume(VolumeId(2), rp).is_err()); + } + #[test] fn test_mount_volume_by_id_hint_skips_lone_sidecar() { // A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a diff --git a/test/fuse_failover/framework_test.go b/test/fuse_failover/framework_test.go index e30c6b3f05..1dd3b59c6d 100644 --- a/test/fuse_failover/framework_test.go +++ b/test/fuse_failover/framework_test.go @@ -110,6 +110,21 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus "mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i))) } require.NoError(t, c.WaitForVolumeServers(60*time.Second)) + + // A mounted filesystem and registered volume servers do not yet guarantee + // that the lock ring has settled or the first replicated volume is ready. + // Exercise the write/read path during setup so the append latency checks + // measure failover on a running cluster rather than its cold start. + probeName := ".failover-ready" + probeData := []byte("ready\n") + probePath := filepath.Join(c.MountDir(0), probeName) + require.NoError(t, os.WriteFile(probePath, probeData, 0644), + "readiness write failed\n%s", c.tailLog("mount0")) + for i := 0; i < numMounts; i++ { + _, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout) + require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i))) + } + require.NoError(t, os.Remove(probePath)) return c } diff --git a/test/fuse_integration/POSIX_COMPLIANCE.md b/test/fuse_integration/POSIX_COMPLIANCE.md new file mode 100644 index 0000000000..f68399b7fb --- /dev/null +++ b/test/fuse_integration/POSIX_COMPLIANCE.md @@ -0,0 +1,645 @@ +# SeaweedFS FUSE POSIX Compliance Testing + +## Overview + +This comprehensive test suite provides full POSIX compliance testing for SeaweedFS FUSE mounts. It includes basic POSIX operations, extended features, external test suite integration, and performance benchmarking to ensure SeaweedFS meets filesystem standards. + +## 🎯 Key Features + +### ✅ **Comprehensive Test Coverage** +- **Basic POSIX Operations**: File/directory create, read, write, delete, rename +- **Advanced Features**: Extended attributes, file locking, memory mapping *(see roadmap)* +- **I/O Operations**: Synchronous/asynchronous I/O, direct I/O, vectored I/O *(see roadmap)* +- **Error Handling**: Comprehensive error condition testing +- **Concurrent Operations**: Multi-threaded stress testing +- **External Integration**: pjdfstest, nfstest, FIO integration + +> **📋 Implementation Status**: Some advanced features are currently skipped pending platform-specific implementations. See [`POSIX_IMPLEMENTATION_ROADMAP.md`](./POSIX_IMPLEMENTATION_ROADMAP.md) for a detailed implementation plan and current status. + +### 📊 **Performance Analysis** +- **Benchmarking**: Built-in performance benchmarks +- **Profiling**: CPU and memory profiling support +- **Coverage Analysis**: Code coverage reporting +- **Stress Testing**: High-load concurrent operation testing + +### 🔧 **External Tool Integration** +- **pjdfstest**: Industry-standard POSIX filesystem test suite +- **nfstest_posix**: Network filesystem POSIX API verification +- **FIO**: Flexible I/O performance testing +- **Custom Litmus Tests**: Focused edge case testing + +## 📋 Test Categories + +### 1. Basic POSIX Compliance (`posix_compliance_test.go`) + +#### File Operations +- ✅ File creation with O_CREAT, O_EXCL flags +- ✅ File truncation and size management +- ✅ File deletion and unlinking +- ✅ Atomic rename operations +- ✅ File permission management + +#### Directory Operations +- ✅ Directory creation and removal +- ✅ Directory listing and traversal +- ✅ Directory rename operations +- ✅ Non-empty directory handling + +#### Symlink Operations +- ✅ Symbolic link creation and resolution +- ✅ Broken symlink handling +- ✅ Symlink target verification +- ✅ Symlink permission handling + +#### Permission Tests +- ✅ File permission setting and verification +- ✅ Directory permission enforcement +- ✅ Permission inheritance +- ✅ Access control validation + +#### Timestamp Tests +- ✅ Access time (atime) updates +- ✅ Modification time (mtime) tracking +- ✅ Change time (ctime) management +- ✅ Timestamp precision and setting + +#### I/O Operations +- ✅ Read/write operations +- ✅ File seeking and positioning +- ✅ Append mode operations +- ✅ Buffer management + +#### File Descriptors +- ✅ File descriptor duplication +- ✅ File descriptor flags +- ✅ Close-on-exec handling +- ✅ File descriptor limits + +#### Atomic Operations +- ✅ Atomic file operations +- ✅ Rename atomicity +- ✅ Link/unlink atomicity + +#### Concurrent Access +- ✅ Multi-reader scenarios +- ✅ Concurrent write handling +- ✅ Reader-writer coordination +- ✅ Race condition prevention + +#### Error Handling +- ✅ ENOENT (file not found) handling +- ✅ EACCES (permission denied) handling +- ✅ EBADF (bad file descriptor) handling +- ✅ ENOTEMPTY (directory not empty) handling +- ✅ Error code compliance + +### 2. Extended POSIX Features (`posix_extended_test.go`) + +#### Extended Attributes +- ✅ Setting extended attributes (setxattr) +- ✅ Getting extended attributes (getxattr) +- ✅ Listing extended attributes (listxattr) +- ✅ Removing extended attributes (removexattr) +- ✅ Extended attribute namespaces + +#### File Locking +- ✅ Advisory locking (fcntl) +- ✅ Exclusive locks (F_WRLCK) +- ✅ Shared locks (F_RDLCK) +- ✅ Lock conflict detection +- ✅ Lock release and cleanup + +#### Advanced I/O +- ✅ Vectored I/O (readv/writev) +- ✅ Positioned I/O (pread/pwrite) +- ✅ Asynchronous I/O patterns +- ✅ Scatter-gather operations + +#### Sparse Files +- ✅ Sparse file creation +- ✅ Hole detection and handling +- ✅ Sparse file efficiency +- ✅ Seek beyond EOF + +#### Large Files (>2GB) +- ✅ Large file operations +- ✅ 64-bit offset handling +- ✅ Large file seeking +- ✅ Large file truncation + +#### Memory Mapping +- ✅ File memory mapping (mmap) +- ✅ Memory-mapped I/O +- ✅ Memory synchronization (msync) +- ✅ Memory unmapping (munmap) +- ✅ Shared/private mappings + +#### Direct I/O +- ✅ Direct I/O operations (O_DIRECT) +- ✅ Buffer alignment requirements +- ✅ Direct I/O performance +- ✅ Bypass buffer cache + +#### File Preallocation +- ✅ Space preallocation (fallocate) +- ✅ File hole punching +- ✅ Space reservation +- ✅ Allocation efficiency + +#### Zero-Copy Operations +- ✅ Zero-copy file transfer (sendfile) +- ✅ Efficient data movement +- ✅ Cross-filesystem transfers + +### 3. External Test Suite Integration (`posix_external_test.go`) + +#### pjdfstest Integration +- ✅ Comprehensive POSIX filesystem test suite +- ✅ Industry-standard compliance verification +- ✅ Automated test execution +- ✅ Result analysis and reporting + +#### nfstest_posix Integration +- ✅ POSIX API verification +- ✅ Network filesystem compliance +- ✅ API behavior validation +- ✅ Comprehensive coverage + +#### Custom Litmus Tests +- ✅ Atomic rename verification +- ✅ Link count accuracy +- ✅ Symlink cycle detection +- ✅ Concurrent operation safety +- ✅ Directory consistency + +#### Stress Testing +- ✅ High-frequency operations +- ✅ Concurrent access patterns +- ✅ Resource exhaustion scenarios +- ✅ Error recovery testing + +#### Edge Case Testing +- ✅ Empty filenames +- ✅ Very long filenames +- ✅ Special characters +- ✅ Deep directory nesting +- ✅ Path length limits + +## 🚀 Quick Start + +### Prerequisites + +```bash +# Install required tools +sudo apt-get install fuse # Linux +brew install macfuse # macOS + +# Install Go 1.21+ +go version + +# Build SeaweedFS +cd /path/to/seaweedfs +make +``` + +### Basic Usage + +```bash +# Navigate to test directory +cd test/fuse_integration + +# Check prerequisites +make -f posix_Makefile check-prereqs + +# Run basic POSIX compliance tests +make -f posix_Makefile test-posix-basic + +# Run complete test suite +make -f posix_Makefile test-posix-full + +# Generate compliance report +make -f posix_Makefile generate-report +``` + +## 🔧 Advanced Usage + +### Test Categories + +#### Critical Tests Only +```bash +# Run only critical POSIX compliance tests (faster) +make -f posix_Makefile test-posix-critical +``` + +#### Extended Feature Tests +```bash +# Test advanced POSIX features +make -f posix_Makefile test-posix-extended +``` + +#### External Tool Integration +```bash +# Setup and run external test suites +make -f posix_Makefile setup-external-tools +make -f posix_Makefile test-posix-external +``` + +#### Stress Testing +```bash +# Run stress tests for concurrent operations +make -f posix_Makefile test-posix-stress +``` + +### Performance Analysis + +#### Benchmarking +```bash +# Run performance benchmarks +make -f posix_Makefile benchmark-posix +``` + +#### Profiling +```bash +# Profile CPU and memory usage +make -f posix_Makefile profile-posix + +# View CPU profile +go tool pprof reports/posix.cpu.prof +``` + +#### Coverage Analysis +```bash +# Generate test coverage report +make -f posix_Makefile coverage-posix +# View: reports/posix_coverage.html +``` + +#### FIO Performance Testing +```bash +# Run FIO-based I/O performance tests +make -f posix_Makefile test-fio-posix +``` + +### External Test Suites + +#### pjdfstest +```bash +# Setup and run pjdfstest +make -f posix_Makefile setup-pjdfstest +make -f posix_Makefile test-pjdfstest +``` + +#### nfstest_posix +```bash +# Install and run nfstest_posix +make -f posix_Makefile setup-nfstest +make -f posix_Makefile test-nfstest-posix +``` + +## 📊 Understanding Results + +### Test Status Indicators + +- ✅ **PASS**: Test completed successfully +- ❌ **FAIL**: Test failed - indicates non-compliance +- ⚠️ **SKIP**: Test skipped - feature not supported +- 🔄 **RETRY**: Test retried due to transient failure + +### Report Formats + +#### HTML Report (`reports/posix_compliance_report.html`) +- Interactive web-based report +- Test category breakdown +- Detailed results with navigation +- Visual status indicators + +#### Text Summary (`reports/posix_compliance_summary.txt`) +- Concise text-based summary +- Key findings and recommendations +- Command-line friendly format + +#### JSON Report (`reports/posix_compliance_report.json`) +- Machine-readable results +- Integration with CI/CD pipelines +- Automated result processing + +### Log Files + +- `posix_basic_results.log`: Basic POSIX test detailed output +- `posix_extended_results.log`: Extended feature test output +- `posix_external_results.log`: External test suite output +- `posix_benchmark_results.log`: Performance benchmark results + +## 🐛 Debugging Failed Tests + +### Common Issues + +#### Permission Denied Errors +```bash +# Ensure proper FUSE permissions +sudo usermod -a -G fuse $USER +# Re-login or run: newgrp fuse + +# Mount with -allowOthers if other users need access (test config MountOptions) +``` + +#### Extended Attributes Not Supported +```bash +# Check if xattr is supported by underlying filesystem +getfattr --version +# May need to skip xattr tests on certain filesystems +``` + +#### File Locking Issues +```bash +# Verify file locking support +# Some network filesystems may not support full locking +``` + +#### Memory Issues +```bash +# Increase available memory for tests +# Large file tests may require significant RAM +``` + +### Debug Mode + +```bash +# Test targets already run go test -v and keep logs under reports/ +make -f posix_Makefile test-posix-basic + +# Run against an existing external mount instead of a fresh cluster +make -f posix_Makefile test-posix-basic TEST_MOUNT_POINT=/path/to/mount + +# Extend the 45-minute per-suite timeout +make -f posix_Makefile test-posix-full TEST_TIMEOUT=90m +``` + +### Manual Test Execution + +```bash +# Run specific test +cd test/fuse_integration +go test -v -run 'TestPOSIXCompliance/FileOperations' . + +# Run with custom timeout +go test -v -timeout 30m -run TestPOSIXExtended . +``` + +## 🔄 CI/CD Integration + +### GitHub Actions + +```yaml +name: POSIX Compliance Tests + +on: [push, pull_request] + +jobs: + posix-compliance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Install FUSE + run: sudo apt-get update && sudo apt-get install -y fuse + + - name: Build SeaweedFS + run: make + + - name: Run POSIX Compliance Tests + run: | + cd test/fuse_integration + make -f posix_Makefile ci-posix-tests + + - name: Upload Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: posix-test-results + path: test/fuse_integration/reports/ +``` + +### Jenkins Pipeline + +```groovy +pipeline { + agent any + stages { + stage('Setup') { + steps { + sh 'make' + } + } + stage('POSIX Compliance') { + steps { + dir('test/fuse_integration') { + sh 'make -f posix_Makefile ci-posix-tests' + } + } + post { + always { + archiveArtifacts 'test/fuse_integration/reports/**' + publishHTML([ + allowMissing: false, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: 'test/fuse_integration/reports', + reportFiles: 'posix_compliance_report.html', + reportName: 'POSIX Compliance Report' + ]) + } + } + } + } +} +``` + +## 🎯 POSIX Compliance Checklist + +### Critical POSIX Features +- [ ] File creation, reading, writing, deletion +- [ ] Directory operations (create, remove, list) +- [ ] File permissions and ownership +- [ ] Symbolic links +- [ ] Hard links +- [ ] File timestamps +- [ ] Error handling compliance +- [ ] Atomic operations + +### Advanced POSIX Features +- [ ] Extended attributes +- [ ] File locking (advisory) +- [ ] Memory-mapped I/O +- [ ] Vectored I/O (readv/writev) +- [ ] Positioned I/O (pread/pwrite) +- [ ] Direct I/O +- [ ] File preallocation +- [ ] Sparse files + +### Performance Requirements +- [ ] Reasonable performance under load +- [ ] Concurrent access handling +- [ ] Memory usage optimization +- [ ] CPU efficiency +- [ ] I/O throughput + +### Reliability Requirements +- [ ] Data consistency +- [ ] Error recovery +- [ ] Race condition prevention +- [ ] Resource cleanup +- [ ] Crash recovery + +## 📈 Performance Benchmarks + +### Expected Performance Characteristics + +| Operation | Expected Range | Notes | +|-----------|----------------|-------| +| File Create | 1000-10000 ops/sec | Depends on chunk size | +| File Read | 100-1000 MB/sec | Network limited | +| File Write | 50-500 MB/sec | Depends on replication | +| Directory Ops | 500-5000 ops/sec | Metadata operations | +| Concurrent Ops | Scales with cores | Up to hardware limits | + +### Benchmarking Commands + +```bash +# Basic I/O performance +make -f posix_Makefile benchmark-posix + +# FIO comprehensive testing +make -f posix_Makefile test-fio-posix + +# Custom benchmark with specific parameters +go test -v -bench=. -benchtime=30s posix_compliance_test.go +``` + +## 🔍 Troubleshooting + +### Common Test Failures + +#### 1. Mount Permission Issues +```bash +# Error: Permission denied mounting +# Solution: Check FUSE permissions — add the user to the fuse group and +# re-login rather than widening permissions on the device node +sudo usermod -a -G fuse $USER +``` + +#### 2. Extended Attribute Tests Fail +```bash +# Error: Operation not supported (ENOTSUP) +# Solution: Check filesystem xattr support +mount | grep $(df . | tail -1 | awk '{print $1}') +# May need to remount with xattr support +``` + +#### 3. File Locking Tests Fail +```bash +# Error: Function not implemented (ENOSYS) +# Solution: Some network filesystems don't support locking +# This may be expected behavior +``` + +#### 4. Large File Tests Fail +```bash +# Error: No space left on device +# Solution: Ensure sufficient disk space +df -h /tmp # Check temp directory space +# May need to clean up or use different temp directory +``` + +#### 5. Performance Tests Timeout +```bash +# Error: Test timeout exceeded +# Solution: Increase timeout or reduce test scope +go test -timeout 60m -run TestPOSIXCompliance +``` + +### Debug Information Collection + +```bash +# Collect system information +uname -a > debug_info.txt +mount >> debug_info.txt +df -h >> debug_info.txt +free -h >> debug_info.txt + +# Collect SeaweedFS information +./weed version >> debug_info.txt +ps aux | grep weed >> debug_info.txt + +# Collect test logs +cp reports/*.log debug_logs/ +``` + +## 📚 Additional Resources + +### POSIX Standards +- [POSIX.1-2017 (IEEE Std 1003.1-2017)](https://pubs.opengroup.org/onlinepubs/9699919799/) +- [Single UNIX Specification](https://www.opengroup.org/openbrand/register/) + +### External Test Suites +- [pjdfstest](https://github.com/pjd/pjdfstest) - Comprehensive POSIX filesystem tests +- [nfstest](https://nfstest.readthedocs.io/) - Network filesystem testing +- [FIO](https://fio.readthedocs.io/) - Flexible I/O tester + +### SeaweedFS Documentation +- [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki) +- [FUSE Mount Documentation](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount) +- [Performance Tuning Guide](https://github.com/seaweedfs/seaweedfs/wiki/Optimization) + +## 🤝 Contributing + +### Adding New Tests + +1. **Basic Tests**: Add to `posix_compliance_test.go` +2. **Extended Tests**: Add to `posix_extended_test.go` +3. **External Integration**: Add to `posix_external_test.go` + +### Test Guidelines + +```go +func (s *POSIXTestSuite) TestNewFeature(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("SubTestName", func(t *testing.T) { + // Test setup + testFile := filepath.Join(mountPoint, "test.txt") + + // Test execution + err := someOperation(testFile) + + // Assertions + require.NoError(t, err) + + // Cleanup (if needed) + defer os.Remove(testFile) + }) +} +``` + +### Submitting Improvements + +1. Fork the repository +2. Create a feature branch +3. Add comprehensive tests +4. Ensure all existing tests pass +5. Submit a pull request with detailed description + +## 📞 Support + +For questions, issues, or contributions: + +- **GitHub Issues**: [SeaweedFS Issues](https://github.com/seaweedfs/seaweedfs/issues) +- **Discussions**: [SeaweedFS Discussions](https://github.com/seaweedfs/seaweedfs/discussions) +- **Documentation**: [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki) + +--- + +*This comprehensive POSIX compliance test suite ensures SeaweedFS FUSE mounts meet industry standards for filesystem behavior, performance, and reliability.* diff --git a/test/fuse_integration/POSIX_IMPLEMENTATION_ROADMAP.md b/test/fuse_integration/POSIX_IMPLEMENTATION_ROADMAP.md new file mode 100644 index 0000000000..5ac9939e72 --- /dev/null +++ b/test/fuse_integration/POSIX_IMPLEMENTATION_ROADMAP.md @@ -0,0 +1,272 @@ +# POSIX Compliance Implementation Roadmap + +This document tracks the implementation status of POSIX features in the SeaweedFS FUSE mount compliance test suite and provides a roadmap for completing comprehensive POSIX compliance testing. + +## Overview + +The platform-specific tests listed below are implemented; this document records the implementation plan that produced them, plus the one feature still pending. + +## Current Status + +### ✅ Implemented Features +- Basic file operations (create, read, write, delete) +- Directory operations (mkdir, rmdir, rename) +- File permissions and ownership +- Symbolic and hard links +- Basic I/O operations (seek, append, positioned I/O) +- File descriptors and atomic operations +- Concurrent access patterns +- Error handling compliance +- Timestamp operations + +### 🚧 Partially Implemented +- Cross-platform compatibility (basic implementation with platform-specific access time handling) +- File sealing (still skipped; Linux-specific, pending) + +### ✅ Implemented Platform-Specific Features + +#### 1. Extended Attributes (xattr) +**Priority: High** +**Platforms: Linux, macOS, FreeBSD** + +**Status:** Implemented +- `TestExtendedAttributes/SetExtendedAttribute` +- `TestExtendedAttributes/ListExtendedAttributes` +- `TestExtendedAttributes/RemoveExtendedAttribute` + +**Implementation Plan:** +```go +// Linux implementation +//go:build linux +func setXattr(path, name string, value []byte) error { + return syscall.Setxattr(path, name, value, 0) +} + +// macOS implementation +//go:build darwin +func setXattr(path, name string, value []byte) error { + return syscall.Setxattr(path, name, value, 0, 0) +} +``` + +**Required Files:** +- `xattr_linux.go` - Linux syscall implementations +- `xattr_darwin.go` - macOS syscall implementations +- `xattr_freebsd.go` - FreeBSD syscall implementations +- `xattr_unsupported.go` - Fallback for unsupported platforms + +#### 2. Memory Mapping (mmap) +**Priority: High** +**Platforms: All POSIX systems** + +**Status:** Implemented +- `TestMemoryMapping/MemoryMappedRead` +- `TestMemoryMapping/MemoryMappedWrite` + +**Implementation Plan:** +```go +func testMmap(t *testing.T, filePath string) { + fd, err := syscall.Open(filePath, syscall.O_RDWR, 0) + require.NoError(t, err) + defer syscall.Close(fd) + + // Platform-specific mmap implementation + data, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) + require.NoError(t, err) + defer syscall.Munmap(data) +} +``` + +**Required Files:** +- `mmap_unix.go` - Unix-like systems implementation +- `mmap_windows.go` - Windows implementation (if needed) + +#### 3. Vectored I/O (readv/writev) +**Priority: Medium** +**Platforms: All POSIX systems** + +**Status:** Implemented +- `TestAdvancedIO/VectoredIO` + +**Implementation Plan:** +```go +func testVectoredIO(t *testing.T, filePath string) { + // Use syscall.Syscall for readv/writev + // Linux: SYS_READV, SYS_WRITEV + // Other platforms: platform-specific syscall numbers +} +``` + +#### 4. Direct I/O +**Priority: Medium** +**Platforms: Linux, some Unix variants** + +**Status:** Implemented +- `TestDirectIO/DirectIO` + +**Implementation Plan:** +```go +//go:build linux +func testDirectIO(t *testing.T, filePath string) { + fd, err := syscall.Open(filePath, syscall.O_RDWR|syscall.O_DIRECT, 0) + // Test with aligned buffers +} +``` + +#### 5. File Preallocation (fallocate) +**Priority: Medium** +**Platforms: Linux, some Unix variants** + +**Status:** Implemented +- `TestFilePreallocation/Fallocate` + +**Implementation Plan:** +```go +//go:build linux +func testFallocate(t *testing.T, filePath string) { + fd, err := syscall.Open(filePath, syscall.O_RDWR, 0) + err = syscall.Syscall6(syscall.SYS_FALLOCATE, uintptr(fd), 0, 0, uintptr(size), 0, 0) +} +``` + +#### 6. Zero-Copy Transfer (sendfile) +**Priority: Low** +**Platforms: Linux, FreeBSD, some Unix variants** + +**Status:** Implemented +- `TestZeroCopyTransfer/Sendfile` + +**Implementation Plan:** +```go +//go:build linux +func testSendfile(t *testing.T, srcPath, dstPath string) { + // Use syscall.Syscall for sendfile + // Platform-specific implementations +} +``` + +#### 7. File Sealing (Linux-specific) +**Priority: Low** +**Platforms: Linux only** + +**Current Status:** File sealing test is still skipped +- `TestFileSealing` + +**Implementation Plan:** +```go +//go:build linux +func testFileSealing(t *testing.T) { + // Use fcntl with F_ADD_SEALS, F_GET_SEALS + // Test various seal types: F_SEAL_WRITE, F_SEAL_SHRINK, etc. +} +``` + +## Implementation Strategy + +### Phase 1: Core Features (High Priority) +1. **Extended Attributes** - Essential for many applications +2. **Memory Mapping** - Critical for performance-sensitive applications + +### Phase 2: Advanced I/O (Medium Priority) +3. **Vectored I/O** - Important for efficient bulk operations +4. **Direct I/O** - Needed for database and high-performance applications +5. **File Preallocation** - Important for preventing fragmentation + +### Phase 3: Specialized Features (Low Priority) +6. **Zero-Copy Transfer** - Performance optimization feature +7. **File Sealing** - Security feature, Linux-specific + +## Platform Support Matrix + +| Feature | Linux | macOS | FreeBSD | Windows | Notes | +|---------|-------|-------|---------|---------|-------| +| Extended Attributes | ✅ | ✅ | ✅ | ❌ | Different syscall signatures | +| Memory Mapping | ✅ | ✅ | ✅ | ✅ | Standard POSIX | +| Vectored I/O | ✅ | ✅ | ✅ | ❌ | readv/writev syscalls | +| Direct I/O | ✅ | ❌ | ✅ | ❌ | O_DIRECT flag | +| fallocate | ✅ | ❌ | ❌ | ❌ | Linux-specific | +| sendfile | ✅ | ❌ | ✅ | ❌ | Platform-specific | +| File Sealing | ✅ | ❌ | ❌ | ❌ | Linux-only | + +## Development Guidelines + +### 1. Platform-Specific Implementation Pattern +```go +// feature_linux.go +//go:build linux +package fuse +func platformSpecificFunction() { /* Linux implementation */ } + +// feature_darwin.go +//go:build darwin +package fuse +func platformSpecificFunction() { /* macOS implementation */ } + +// feature_unsupported.go +//go:build !linux && !darwin +package fuse +func platformSpecificFunction() { /* Skip or error */ } +``` + +### 2. Test Structure Pattern +```go +func (s *POSIXExtendedTestSuite) TestFeature(t *testing.T) { + if !isFeatureSupported() { + t.Skip("Feature not supported on this platform") + return + } + + t.Run("FeatureTest", func(t *testing.T) { + // Actual test implementation + }) +} +``` + +### 3. Error Handling +- Use platform-specific error checking +- Provide meaningful error messages +- Gracefully handle unsupported features + +## Testing Strategy + +### 1. Continuous Integration +- Run tests on multiple platforms (Linux, macOS) +- Use build tags to enable/disable platform-specific tests +- Ensure graceful degradation on unsupported platforms + +### 2. Feature Detection +- Implement runtime feature detection where possible +- Skip tests gracefully when features are unavailable +- Log clear messages about skipped functionality + +### 3. Documentation +- Document platform-specific behavior +- Provide examples for each implemented feature +- Maintain compatibility matrices + +## Contributing + +When implementing these features: + +1. **Start with high-priority items** (Extended Attributes, Memory Mapping) +2. **Follow the platform-specific pattern** outlined above +3. **Add comprehensive tests** for each feature +4. **Update this roadmap** as features are implemented +5. **Document any platform-specific quirks** or limitations + +## Future Enhancements + +Beyond the current roadmap, consider: +- **File locking** (flock, fcntl locks) +- **Asynchronous I/O** (aio_read, aio_write) +- **File change notifications** (inotify, kqueue) +- **POSIX ACLs** (Access Control Lists) +- **Sparse file operations** +- **File hole punching** + +## References + +- [POSIX.1-2017 Standard](https://pubs.opengroup.org/onlinepubs/9699919799/) +- [Linux Programmer's Manual](https://man7.org/linux/man-pages/) +- [FreeBSD System Calls](https://www.freebsd.org/cgi/man.cgi) +- [macOS System Calls](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/) diff --git a/test/fuse_integration/atime_linux.go b/test/fuse_integration/atime_linux.go new file mode 100644 index 0000000000..9d8e9579ae --- /dev/null +++ b/test/fuse_integration/atime_linux.go @@ -0,0 +1,12 @@ +//go:build linux + +package fuse + +import ( + "syscall" +) + +// getAtimeNano returns the access time in nanoseconds for Linux systems +func getAtimeNano(stat *syscall.Stat_t) int64 { + return stat.Atim.Sec*1e9 + stat.Atim.Nsec +} diff --git a/test/fuse_integration/atime_nonlinux.go b/test/fuse_integration/atime_nonlinux.go new file mode 100644 index 0000000000..c3b7cda16a --- /dev/null +++ b/test/fuse_integration/atime_nonlinux.go @@ -0,0 +1,12 @@ +//go:build !linux && !windows + +package fuse + +import ( + "syscall" +) + +// getAtimeNano returns the access time in nanoseconds for non-Linux systems (macOS, BSD, etc.) +func getAtimeNano(stat *syscall.Stat_t) int64 { + return stat.Atimespec.Sec*1e9 + stat.Atimespec.Nsec +} diff --git a/test/fuse_integration/atime_windows.go b/test/fuse_integration/atime_windows.go new file mode 100644 index 0000000000..4b8a7d9c6d --- /dev/null +++ b/test/fuse_integration/atime_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +package fuse + +import ( + "syscall" +) + +// getAtimeNano returns the access time in nanoseconds for Windows systems +func getAtimeNano(stat *syscall.Win32FileAttributeData) int64 { + return stat.LastAccessTime.Nanoseconds() +} diff --git a/test/fuse_integration/directio_darwin.go b/test/fuse_integration/directio_darwin.go new file mode 100644 index 0000000000..67ccaf9a03 --- /dev/null +++ b/test/fuse_integration/directio_darwin.go @@ -0,0 +1,39 @@ +//go:build darwin + +package fuse + +import ( + "syscall" +) + +// Direct I/O support for macOS + +const ( + // macOS doesn't have O_DIRECT, but we can use fcntl with F_NOCACHE + F_NOCACHE = 48 +) + +func openDirectIO(path string, flags int, mode uint32) (int, error) { + // Open file normally first + fd, err := syscall.Open(path, flags, mode) + if err != nil { + return -1, err + } + + // Set F_NOCACHE to bypass buffer cache (similar to O_DIRECT) + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, + uintptr(fd), + F_NOCACHE, + 1) // enable + + if errno != 0 { + syscall.Close(fd) + return -1, errno + } + + return fd, nil +} + +func isDirectIOSupported() bool { + return true +} diff --git a/test/fuse_integration/directio_linux.go b/test/fuse_integration/directio_linux.go new file mode 100644 index 0000000000..1ade7474f5 --- /dev/null +++ b/test/fuse_integration/directio_linux.go @@ -0,0 +1,23 @@ +//go:build linux + +package fuse + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +// Direct I/O support for Linux + +const ( + O_DIRECT = unix.O_DIRECT +) + +func openDirectIO(path string, flags int, mode uint32) (int, error) { + return syscall.Open(path, flags|O_DIRECT, mode) +} + +func isDirectIOSupported() bool { + return true +} diff --git a/test/fuse_integration/directio_unsupported.go b/test/fuse_integration/directio_unsupported.go new file mode 100644 index 0000000000..dc1168b1cc --- /dev/null +++ b/test/fuse_integration/directio_unsupported.go @@ -0,0 +1,29 @@ +//go:build !linux && !darwin + +package fuse + +import ( + "errors" + "syscall" +) + +// Direct I/O support for unsupported platforms + +var ErrDirectIONotSupported = errors.New("direct I/O not supported on this platform") + +const ( + O_DIRECT = 0x4000 // Dummy flag for compatibility +) + +func openDirectIO(path string, flags int, mode uint32) (int, error) { + // Fall back to regular open + fd, err := syscall.Open(path, flags, mode) + if err != nil { + return -1, err + } + return int(fd), nil +} + +func isDirectIOSupported() bool { + return false +} diff --git a/test/fuse_integration/fallocate_darwin.go b/test/fuse_integration/fallocate_darwin.go new file mode 100644 index 0000000000..fa67f3f7a3 --- /dev/null +++ b/test/fuse_integration/fallocate_darwin.go @@ -0,0 +1,79 @@ +//go:build darwin + +package fuse + +import ( + "syscall" + "unsafe" +) + +// File allocation support for macOS using fcntl + +const ( + // macOS doesn't have fallocate, but we can use fcntl with F_PREALLOCATE + F_ALLOCATECONTIG = 0x02 + F_ALLOCATEALL = 0x04 + + // Dummy flags for compatibility + FALLOC_FL_KEEP_SIZE = 0x01 + FALLOC_FL_PUNCH_HOLE = 0x02 + FALLOC_FL_NO_HIDE_STALE = 0x04 + FALLOC_FL_COLLAPSE_RANGE = 0x08 + FALLOC_FL_ZERO_RANGE = 0x10 + FALLOC_FL_INSERT_RANGE = 0x20 + FALLOC_FL_UNSHARE_RANGE = 0x40 +) + +// fstore_t structure for F_PREALLOCATE +type fstore struct { + flags uint32 + posmode int16 + offset int64 + length int64 + bytesalloc int64 +} + +func fallocateFile(fd int, mode int, offset int64, length int64) error { + // Check for unsupported modes on macOS + unsupportedModes := FALLOC_FL_PUNCH_HOLE | FALLOC_FL_NO_HIDE_STALE | FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE + if mode&unsupportedModes != 0 { + return syscall.ENOTSUP // Operation not supported + } + + // On macOS, we use fcntl with F_PREALLOCATE + store := fstore{ + flags: F_ALLOCATECONTIG, + posmode: syscall.F_PEOFPOSMODE, // Allocate from EOF + offset: 0, + length: length, + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, + uintptr(fd), + syscall.F_PREALLOCATE, + uintptr(unsafe.Pointer(&store))) + + if errno != 0 { + // If contiguous allocation fails, try non-contiguous + store.flags = F_ALLOCATEALL + _, _, errno = syscall.Syscall(syscall.SYS_FCNTL, + uintptr(fd), + syscall.F_PREALLOCATE, + uintptr(unsafe.Pointer(&store))) + + if errno != 0 { + return errno + } + } + + // Set file size if not keeping size + if mode&FALLOC_FL_KEEP_SIZE == 0 { + return syscall.Ftruncate(fd, offset+length) + } + + return nil +} + +func isFallocateSupported() bool { + return true +} diff --git a/test/fuse_integration/fallocate_linux.go b/test/fuse_integration/fallocate_linux.go new file mode 100644 index 0000000000..7b1feaa4e6 --- /dev/null +++ b/test/fuse_integration/fallocate_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +package fuse + +import ( + "syscall" +) + +// File allocation support for Linux + +const ( + // Fallocate flags + FALLOC_FL_KEEP_SIZE = 0x01 + FALLOC_FL_PUNCH_HOLE = 0x02 + FALLOC_FL_NO_HIDE_STALE = 0x04 + FALLOC_FL_COLLAPSE_RANGE = 0x08 + FALLOC_FL_ZERO_RANGE = 0x10 + FALLOC_FL_INSERT_RANGE = 0x20 + FALLOC_FL_UNSHARE_RANGE = 0x40 +) + +func fallocateFile(fd int, mode int, offset int64, length int64) error { + _, _, errno := syscall.Syscall6(syscall.SYS_FALLOCATE, + uintptr(fd), + uintptr(mode), + uintptr(offset), + uintptr(length), + 0, 0) + + if errno != 0 { + return errno + } + return nil +} + +func isFallocateSupported() bool { + return true +} diff --git a/test/fuse_integration/fallocate_unsupported.go b/test/fuse_integration/fallocate_unsupported.go new file mode 100644 index 0000000000..8be54785d0 --- /dev/null +++ b/test/fuse_integration/fallocate_unsupported.go @@ -0,0 +1,30 @@ +//go:build !linux && !darwin + +package fuse + +import ( + "errors" +) + +// File allocation support for unsupported platforms + +var ErrFallocateNotSupported = errors.New("fallocate not supported on this platform") + +const ( + // Dummy flags for compatibility + FALLOC_FL_KEEP_SIZE = 0x01 + FALLOC_FL_PUNCH_HOLE = 0x02 + FALLOC_FL_NO_HIDE_STALE = 0x04 + FALLOC_FL_COLLAPSE_RANGE = 0x08 + FALLOC_FL_ZERO_RANGE = 0x10 + FALLOC_FL_INSERT_RANGE = 0x20 + FALLOC_FL_UNSHARE_RANGE = 0x40 +) + +func fallocateFile(fd int, mode int, offset int64, length int64) error { + return ErrFallocateNotSupported +} + +func isFallocateSupported() bool { + return false +} diff --git a/test/fuse_integration/framework.go b/test/fuse_integration/framework.go new file mode 100644 index 0000000000..8424fb1d79 --- /dev/null +++ b/test/fuse_integration/framework.go @@ -0,0 +1,496 @@ +package fuse + +import ( + "fmt" + "io/fs" + "net" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// FuseTestFramework provides utilities for FUSE integration testing +type FuseTestFramework struct { + t *testing.T + tempDir string + mountPoint string + dataDir string + masterProcess *os.Process + volumeProcess *os.Process + filerProcess *os.Process + mountProcess *os.Process + masterAddr string + volumeAddr string + filerAddr string + weedBinary string + isSetup bool + config *TestConfig + externalMount bool +} + +// TestConfig holds configuration for FUSE tests +type TestConfig struct { + Collection string + Replication string + ChunkSizeMB int + CacheSizeMB int + NumVolumes int + EnableDebug bool + MountOptions []string + SkipCleanup bool // for debugging failed tests +} + +// DefaultTestConfig returns a default configuration for FUSE tests +func DefaultTestConfig() *TestConfig { + return &TestConfig{ + Collection: "", + Replication: "000", + ChunkSizeMB: 4, + CacheSizeMB: 100, + NumVolumes: 3, + EnableDebug: false, + MountOptions: []string{}, + SkipCleanup: false, + } +} + +// NewFuseTestFramework creates a new FUSE testing framework +func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { + if config == nil { + config = DefaultTestConfig() + } + + tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_") + require.NoError(t, err) + + return &FuseTestFramework{ + t: t, + tempDir: tempDir, + mountPoint: filepath.Join(tempDir, "mount"), + dataDir: filepath.Join(tempDir, "data"), + masterAddr: "127.0.0.1:19333", + volumeAddr: "127.0.0.1:18080", + filerAddr: "127.0.0.1:18888", + weedBinary: findWeedBinary(), + isSetup: false, + config: config, + } +} + +// Setup starts SeaweedFS cluster and mounts FUSE filesystem +func (f *FuseTestFramework) Setup(config *TestConfig) error { + if f.isSetup { + return fmt.Errorf("framework already setup") + } + if config != nil { + f.config = config + } + config = f.config + + // Check if we should skip cluster setup and use existing mount + if os.Getenv("TEST_SKIP_CLUSTER_SETUP") == "true" { + // Use existing mount point from environment + if existingMount := os.Getenv("TEST_MOUNT_POINT"); existingMount != "" { + f.externalMount = true + + // Verify mount point is accessible + if _, err := os.Stat(existingMount); err != nil { + return fmt.Errorf("existing mount point not accessible: %v", err) + } + + // A plain directory passes every check below; refuse to run the suite + // against the local disk + if !mountedAt(existingMount) { + return fmt.Errorf("mount point %s is not a mounted filesystem", existingMount) + } + // So can a non-FUSE mount like tmpfs: a device boundary alone does + // not make the results FUSE compliance + if !fuseFilesystem(existingMount) { + return fmt.Errorf("mount point %s is not a FUSE filesystem", existingMount) + } + + // Run inside a private work dir so test files do not pile up in + // the shared mount root and cleanup can remove only what it made + workDir, err := os.MkdirTemp(existingMount, ".fuse_test_") + if err != nil { + return fmt.Errorf("creating work dir under existing mount: %v", err) + } + f.mountPoint = workDir + f.t.Logf("Using existing mount point: %s (work dir %s)", existingMount, f.mountPoint) + + // Test basic functionality + testFile := filepath.Join(f.mountPoint, ".framework_test") + if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil { + return fmt.Errorf("mount point not writable: %v", err) + } + if err := os.Remove(testFile); err != nil { + f.t.Logf("Warning: failed to cleanup test file: %v", err) + } + + // Verify this is actually a SeaweedFS mount by checking for SeaweedFS-specific behavior + // Create a test file and verify it appears in the filer + verifyFile := filepath.Join(f.mountPoint, ".seaweedfs_mount_verification") + if err := os.WriteFile(verifyFile, []byte("SeaweedFS mount verification"), 0644); err != nil { + return fmt.Errorf("mount point verification failed - cannot write: %v", err) + } + + // Read it back to ensure it's working + if data, err := os.ReadFile(verifyFile); err != nil { + return fmt.Errorf("mount point verification failed - cannot read: %v", err) + } else if string(data) != "SeaweedFS mount verification" { + return fmt.Errorf("mount point verification failed - data mismatch") + } + + // Clean up verification file + if err := os.Remove(verifyFile); err != nil { + f.t.Logf("Warning: failed to cleanup verification file: %v", err) + } + + f.t.Logf("✅ SeaweedFS mount point verified and working: %s", f.mountPoint) + + f.isSetup = true + return nil + } + } + + // Create directories for full cluster setup + dirs := []string{f.mountPoint, f.dataDir} + for _, dir := range dirs { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", dir, err) + } + } + + // Start master + if err := f.startMaster(config); err != nil { + return fmt.Errorf("failed to start master: %v", err) + } + + // Wait for master to be ready + if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil { + return fmt.Errorf("master not ready: %v", err) + } + + // Start volume servers + if err := f.startVolumeServers(config); err != nil { + return fmt.Errorf("failed to start volume servers: %v", err) + } + + // Wait for volume server to be ready + if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil { + return fmt.Errorf("volume server not ready: %v", err) + } + + // Start filer + if err := f.startFiler(config); err != nil { + return fmt.Errorf("failed to start filer: %v", err) + } + + // Wait for filer to be ready + if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil { + return fmt.Errorf("filer not ready: %v", err) + } + + // Skip rather than fail on a host with no FUSE device to mount through + if !fuseAvailable() { + f.t.Skip("no FUSE device node (/dev/fuse, /dev/macfuse0, /dev/osxfuse0)") + } + + // Mount FUSE filesystem + if err := f.mountFuse(config); err != nil { + return fmt.Errorf("failed to mount FUSE: %v", err) + } + + // Wait for mount to be ready + if err := f.waitForMount(30 * time.Second); err != nil { + return fmt.Errorf("FUSE mount not ready: %v", err) + } + + f.isSetup = true + return nil +} + +// Cleanup stops all processes and removes temporary files +func (f *FuseTestFramework) Cleanup() { + // An external mount needs no teardown; only this run's work dir is ours. + // A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster + // still owns its processes and must not skip stopping them. + if f.externalMount { + if !f.config.SkipCleanup { + os.RemoveAll(f.mountPoint) + os.RemoveAll(f.tempDir) + } + return + } + + if f.mountProcess != nil { + f.unmountFuse() + } + + // Stop processes in reverse order + processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess} + for _, proc := range processes { + if proc != nil { + proc.Signal(syscall.SIGTERM) + proc.Wait() + } + } + + // Remove temp directory + if !f.config.SkipCleanup { + os.RemoveAll(f.tempDir) + } +} + +// GetMountPoint returns the FUSE mount point path +func (f *FuseTestFramework) GetMountPoint() string { + return f.mountPoint +} + +// GetFilerAddr returns the filer address +func (f *FuseTestFramework) GetFilerAddr() string { + return f.filerAddr +} + +// startMaster starts the SeaweedFS master server +func (f *FuseTestFramework) startMaster(config *TestConfig) error { + var args []string + if config.EnableDebug { + // -v is a global flag; before the subcommand or flag parsing dies + args = append(args, "-v=4") + } + args = append(args, + "master", + "-ip=127.0.0.1", + "-port=19333", + "-mdir=" + filepath.Join(f.dataDir, "master"), + "-raftBootstrap", + ) + + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + if err := cmd.Start(); err != nil { + return err + } + f.masterProcess = cmd.Process + return nil +} + +// startVolumeServers starts SeaweedFS volume servers +func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error { + var args []string + if config.EnableDebug { + args = append(args, "-v=4") + } + args = append(args, + "volume", + "-mserver=" + f.masterAddr, + "-ip=127.0.0.1", + "-port=18080", + "-dir=" + filepath.Join(f.dataDir, "volume"), + fmt.Sprintf("-max=%d", config.NumVolumes), + ) + + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + if err := cmd.Start(); err != nil { + return err + } + f.volumeProcess = cmd.Process + return nil +} + +// startFiler starts the SeaweedFS filer server +func (f *FuseTestFramework) startFiler(config *TestConfig) error { + var args []string + if config.EnableDebug { + args = append(args, "-v=4") + } + args = append(args, + "filer", + "-master=" + f.masterAddr, + "-ip=127.0.0.1", + "-port=18888", + ) + + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + if err := cmd.Start(); err != nil { + return err + } + f.filerProcess = cmd.Process + return nil +} + +// mountFuse mounts the SeaweedFS FUSE filesystem +func (f *FuseTestFramework) mountFuse(config *TestConfig) error { + var args []string + if config.EnableDebug { + args = append(args, "-v=4") + } + args = append(args, + "mount", + "-filer=" + f.filerAddr, + "-dir=" + f.mountPoint, + "-filer.path=/", + "-dirAutoCreate", + ) + + if config.Collection != "" { + args = append(args, "-collection="+config.Collection) + } + if config.Replication != "" { + args = append(args, "-replication="+config.Replication) + } + if config.ChunkSizeMB > 0 { + args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB)) + } + if config.CacheSizeMB > 0 { + args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB)) + } + + args = append(args, config.MountOptions...) + + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + if err := cmd.Start(); err != nil { + return err + } + f.mountProcess = cmd.Process + return nil +} + +// unmountFuse unmounts the FUSE filesystem +func (f *FuseTestFramework) unmountFuse() error { + if f.mountProcess != nil { + f.mountProcess.Signal(syscall.SIGTERM) + f.mountProcess.Wait() + f.mountProcess = nil + } + + // Also try system unmount as backup + exec.Command("umount", f.mountPoint).Run() + return nil +} + +// waitForService waits for a service to be available +func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 1*time.Second) + if err == nil { + conn.Close() + return nil + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("service at %s not ready within timeout", addr) +} + +// waitForMount waits for the FUSE mount to be ready +func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { + // The mount point is created before weed mount starts, so Stat/ReadDir + // alone succeed on the plain directory. The device id only differs from + // its parent's once a filesystem is actually mounted on it. + parentDev, err := deviceID(filepath.Dir(f.mountPoint)) + if err != nil { + return fmt.Errorf("stat mount parent: %w", err) + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if dev, err := deviceID(f.mountPoint); err == nil && dev != parentDev { + if _, err := os.ReadDir(f.mountPoint); err == nil { + return nil + } + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("mount point not ready within timeout") +} + +// fuseAvailable reports whether the host has a FUSE device node to mount +// through: /dev/fuse on Linux, /dev/macfuse* or /dev/osxfuse* on macOS. +func fuseAvailable() bool { + for _, dev := range []string{"/dev/fuse", "/dev/macfuse0", "/dev/osxfuse0"} { + if _, err := os.Stat(dev); err == nil { + return true + } + } + return false +} + +// findWeedBinary locates the weed binary. Candidates are resolved to +// absolute paths because the started processes run with cmd.Dir set to the +// framework's temp dir, where a relative path would not resolve. +func findWeedBinary() string { + if p, err := exec.LookPath("weed"); err == nil { + return p + } + + candidates := []string{ + "../../weed/weed", + "./weed", + "../weed", + } + for _, candidate := range candidates { + if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0111 != 0 { + if abs, err := filepath.Abs(candidate); err == nil { + return abs + } + } + } + + // Default fallback + return "weed" +} + +// Helper functions for test assertions + +// AssertFileExists checks if a file exists in the mount point +func (f *FuseTestFramework) AssertFileExists(relativePath string) { + fullPath := filepath.Join(f.mountPoint, relativePath) + _, err := os.Stat(fullPath) + require.NoError(f.t, err, "file should exist: %s", relativePath) +} + +// AssertFileNotExists checks if a file does not exist in the mount point +func (f *FuseTestFramework) AssertFileNotExists(relativePath string) { + fullPath := filepath.Join(f.mountPoint, relativePath) + _, err := os.Stat(fullPath) + require.True(f.t, os.IsNotExist(err), "file should not exist: %s", relativePath) +} + +// AssertFileContent checks if a file has expected content +func (f *FuseTestFramework) AssertFileContent(relativePath string, expectedContent []byte) { + fullPath := filepath.Join(f.mountPoint, relativePath) + actualContent, err := os.ReadFile(fullPath) + require.NoError(f.t, err, "failed to read file: %s", relativePath) + require.Equal(f.t, expectedContent, actualContent, "file content mismatch: %s", relativePath) +} + +// AssertFileMode checks if a file has expected permissions +func (f *FuseTestFramework) AssertFileMode(relativePath string, expectedMode fs.FileMode) { + fullPath := filepath.Join(f.mountPoint, relativePath) + info, err := os.Stat(fullPath) + require.NoError(f.t, err, "failed to stat file: %s", relativePath) + require.Equal(f.t, expectedMode, info.Mode(), "file mode mismatch: %s", relativePath) +} + +// CreateTestFile creates a test file with specified content +func (f *FuseTestFramework) CreateTestFile(relativePath string, content []byte) { + fullPath := filepath.Join(f.mountPoint, relativePath) + dir := filepath.Dir(fullPath) + require.NoError(f.t, os.MkdirAll(dir, 0755), "failed to create directory: %s", dir) + require.NoError(f.t, os.WriteFile(fullPath, content, 0644), "failed to create file: %s", relativePath) +} + +// CreateTestDir creates a test directory +func (f *FuseTestFramework) CreateTestDir(relativePath string) { + fullPath := filepath.Join(f.mountPoint, relativePath) + require.NoError(f.t, os.MkdirAll(fullPath, 0755), "failed to create directory: %s", relativePath) +} diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index ec0fc19b5a..23af7e9dd2 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -30,6 +30,7 @@ type FuseTestFramework struct { filerPort int weedBinary string isSetup bool + config *TestConfig } // TestConfig holds configuration for FUSE tests @@ -83,6 +84,7 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort), weedBinary: findWeedBinary(), isSetup: false, + config: config, } } @@ -132,6 +134,10 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { if f.isSetup { return fmt.Errorf("framework already setup") } + if config != nil { + f.config = config + } + config = f.config dirs := []string{f.mountPoint, f.logDir, f.dataDir} for _, dir := range dirs { @@ -182,7 +188,7 @@ func (f *FuseTestFramework) Cleanup() { f.copyLogsForCI() - if !DefaultTestConfig().SkipCleanup { + if !f.config.SkipCleanup { os.RemoveAll(f.tempDir) } } diff --git a/test/fuse_integration/fuse_mount_bsd.go b/test/fuse_integration/fuse_mount_bsd.go new file mode 100644 index 0000000000..d1d731ad76 --- /dev/null +++ b/test/fuse_integration/fuse_mount_bsd.go @@ -0,0 +1,25 @@ +//go:build darwin || freebsd + +package fuse + +import ( + "strings" + "syscall" +) + +// fuseFilesystem reports whether path sits on a FUSE mount; macFUSE, osxfuse +// and fusefs all carry "fuse" in the filesystem type name. +func fuseFilesystem(path string) bool { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return false + } + name := make([]byte, 0, len(st.Fstypename)) + for _, c := range st.Fstypename { + if c == 0 { + break + } + name = append(name, byte(c)) + } + return strings.Contains(string(name), "fuse") +} diff --git a/test/fuse_integration/fuse_mount_linux.go b/test/fuse_integration/fuse_mount_linux.go new file mode 100644 index 0000000000..2a181cb4ce --- /dev/null +++ b/test/fuse_integration/fuse_mount_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package fuse + +import "syscall" + +// fuseSuperMagic is statfs f_type for fuse and fuseblk mounts. +const fuseSuperMagic = 0x65735546 + +// fuseFilesystem reports whether path sits on a FUSE mount. +func fuseFilesystem(path string) bool { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return false + } + return st.Type == fuseSuperMagic +} diff --git a/test/fuse_integration/fuse_mount_other.go b/test/fuse_integration/fuse_mount_other.go new file mode 100644 index 0000000000..21022f9a5c --- /dev/null +++ b/test/fuse_integration/fuse_mount_other.go @@ -0,0 +1,9 @@ +//go:build (unix && !linux && !darwin && !freebsd) || (!unix && !windows) + +package fuse + +// fuseFilesystem has no portable check on these platforms; the mount check +// and write verification in Setup stay the only gates. +func fuseFilesystem(path string) bool { + return true +} diff --git a/test/fuse_integration/mmap_unix.go b/test/fuse_integration/mmap_unix.go new file mode 100644 index 0000000000..4fa9619c1e --- /dev/null +++ b/test/fuse_integration/mmap_unix.go @@ -0,0 +1,55 @@ +//go:build unix + +package fuse + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +// Memory mapping support for Unix-like systems + +func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) { + return unix.Mmap(fd, offset, length, prot, flags) +} + +func munmapFile(data []byte) error { + if len(data) == 0 { + return nil + } + return unix.Munmap(data) +} + +func msyncFile(data []byte, flags int) error { + if len(data) == 0 { + return nil + } + return unix.Msync(data, flags) +} + +func isMmapSupported() bool { + return true +} + +// Memory protection flags +const ( + PROT_READ = syscall.PROT_READ + PROT_WRITE = syscall.PROT_WRITE + PROT_EXEC = syscall.PROT_EXEC + PROT_NONE = syscall.PROT_NONE +) + +// Memory mapping flags +const ( + MAP_SHARED = syscall.MAP_SHARED + MAP_PRIVATE = syscall.MAP_PRIVATE + MAP_ANONYMOUS = syscall.MAP_ANON +) + +// Memory sync flags +const ( + MS_ASYNC = syscall.MS_ASYNC + MS_SYNC = syscall.MS_SYNC + MS_INVALIDATE = syscall.MS_INVALIDATE +) diff --git a/test/fuse_integration/mmap_unsupported.go b/test/fuse_integration/mmap_unsupported.go new file mode 100644 index 0000000000..082f9333c4 --- /dev/null +++ b/test/fuse_integration/mmap_unsupported.go @@ -0,0 +1,47 @@ +//go:build !unix + +package fuse + +import ( + "errors" +) + +// Memory mapping support for unsupported platforms + +var ErrMmapNotSupported = errors.New("memory mapping not supported on this platform") + +func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) { + return nil, ErrMmapNotSupported +} + +func munmapFile(data []byte) error { + return ErrMmapNotSupported +} + +func msyncFile(data []byte, flags int) error { + return ErrMmapNotSupported +} + +func isMmapSupported() bool { + return false +} + +// Dummy constants for unsupported platforms +const ( + PROT_READ = 0x1 + PROT_WRITE = 0x2 + PROT_EXEC = 0x4 + PROT_NONE = 0x0 +) + +const ( + MAP_SHARED = 0x01 + MAP_PRIVATE = 0x02 + MAP_ANONYMOUS = 0x20 +) + +const ( + MS_ASYNC = 0x1 + MS_SYNC = 0x4 + MS_INVALIDATE = 0x2 +) diff --git a/test/fuse_integration/mounted_linux_test.go b/test/fuse_integration/mounted_linux_test.go new file mode 100644 index 0000000000..c2184cafff --- /dev/null +++ b/test/fuse_integration/mounted_linux_test.go @@ -0,0 +1,27 @@ +//go:build linux + +package fuse + +import "testing" + +func TestMountedAt(t *testing.T) { + for _, p := range []string{"/proc", "/proc/"} { + if !mountedAt(p) { + t.Errorf("mountedAt(%q) = false, want true", p) + } + } + plain := t.TempDir() + for _, p := range []string{plain, plain + "/"} { + if mountedAt(p) { + t.Errorf("mountedAt(%q) = true, want false", p) + } + } +} + +func TestFuseFilesystem(t *testing.T) { + for _, p := range []string{"/proc", t.TempDir()} { + if fuseFilesystem(p) { + t.Errorf("fuseFilesystem(%q) = true, want false", p) + } + } +} diff --git a/test/fuse_integration/mounted_unix.go b/test/fuse_integration/mounted_unix.go new file mode 100644 index 0000000000..471b8bb655 --- /dev/null +++ b/test/fuse_integration/mounted_unix.go @@ -0,0 +1,33 @@ +//go:build unix + +package fuse + +import ( + "path/filepath" + "syscall" +) + +// deviceID returns the device ID of the filesystem containing path. +func deviceID(path string) (uint64, error) { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return 0, err + } + return uint64(st.Dev), nil +} + +// mountedAt reports whether something is actually mounted at path: a plain +// directory shares its parent's device, a mount point does not. The path is +// cleaned first so a trailing slash does not make Dir return the mount itself. +func mountedAt(path string) bool { + path = filepath.Clean(path) + dev, err := deviceID(path) + if err != nil { + return false + } + parentDev, err := deviceID(filepath.Dir(path)) + if err != nil { + return false + } + return dev != parentDev +} diff --git a/test/fuse_integration/mounted_unsupported.go b/test/fuse_integration/mounted_unsupported.go new file mode 100644 index 0000000000..1d698e5241 --- /dev/null +++ b/test/fuse_integration/mounted_unsupported.go @@ -0,0 +1,16 @@ +//go:build !unix && !windows + +package fuse + +// FUSE mounts are unix-only: reporting one device keeps waitForMount from +// mistaking the plain pre-mount directory for a live mount on platforms +// where a mount can never appear. +func deviceID(path string) (uint64, error) { + return 0, nil +} + +// mountedAt cannot be detected off unix; report mounted so an external +// TEST_MOUNT_POINT keeps its best-effort write verification. +func mountedAt(path string) bool { + return true +} diff --git a/test/fuse_integration/mounted_windows.go b/test/fuse_integration/mounted_windows.go new file mode 100644 index 0000000000..4855bb647b --- /dev/null +++ b/test/fuse_integration/mounted_windows.go @@ -0,0 +1,40 @@ +//go:build windows + +package fuse + +import ( + "os" + "path/filepath" + "syscall" +) + +// deviceID has no portable meaning here: reporting one device keeps +// waitForMount from mistaking the plain pre-mount directory for a live mount. +func deviceID(path string) (uint64, error) { + return 0, nil +} + +// mountedAt reports whether path is a WinFsp mount: a directory mount is a +// reparse point and a drive mount is a volume root, while a plain directory +// is neither. +func mountedAt(path string) bool { + abs, err := filepath.Abs(path) + if err != nil { + return false + } + if filepath.Dir(abs) == abs { + return true + } + info, err := os.Lstat(abs) + if err != nil { + return false + } + data, ok := info.Sys().(*syscall.Win32FileAttributeData) + return ok && data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 +} + +// fuseFilesystem is covered by the WinFsp checks in mountedAt: WinFsp is the +// only FUSE provider on the platform. +func fuseFilesystem(path string) bool { + return true +} diff --git a/test/fuse_integration/posix_Makefile b/test/fuse_integration/posix_Makefile new file mode 100644 index 0000000000..32c5c2f45f --- /dev/null +++ b/test/fuse_integration/posix_Makefile @@ -0,0 +1,531 @@ +# SeaweedFS POSIX Compliance Testing Makefile + +SHELL := /bin/bash +# go test is piped through tee everywhere; without pipefail a failing suite +# still exits 0 because only tee's status propagates. +.SHELLFLAGS := -o pipefail -c + +# Configuration +WEED_BINARY := $(shell which weed 2>/dev/null || echo "../../weed") +GO_VERSION := 1.21 +TEST_TIMEOUT := 45m +COVERAGE_FILE := posix_coverage.out +REPORT_DIR := reports +EXTERNAL_TOOLS_DIR := external_tools +MOUNT_POINT ?= /tmp/seaweedfs_mount +ifneq ($(TEST_MOUNT_POINT),) +MOUNT_POINT := $(TEST_MOUNT_POINT) +endif + +# Go test command prefix for external mount points +GO_TEST_PREFIX := +ifneq ($(TEST_MOUNT_POINT),) +GO_TEST_PREFIX := TEST_MOUNT_POINT="$(TEST_MOUNT_POINT)" TEST_SKIP_CLUSTER_SETUP="true" +endif + +# Test categories +POSIX_BASIC_TESTS := posix_compliance_test.go +POSIX_EXTENDED_TESTS := posix_extended_test.go +POSIX_EXTERNAL_TESTS := posix_external_test.go + +# -f is not propagated through MAKEFLAGS, so recursive calls must name this +# file explicitly or the sub-make reads the default Makefile instead. +MAKE_SELF := $(MAKE) -f $(firstword $(MAKEFILE_LIST)) + +# Colors for output — bash echo does not expand \033, so bake the escapes in +RED := $(shell printf '\033[31m') +GREEN := $(shell printf '\033[32m') +YELLOW := $(shell printf '\033[33m') +BLUE := $(shell printf '\033[34m') +MAGENTA := $(shell printf '\033[35m') +CYAN := $(shell printf '\033[36m') +WHITE := $(shell printf '\033[37m') +RESET := $(shell printf '\033[0m') + +.DEFAULT_GOAL := help + +# Prerequisites checks +check-binary: + @if [ ! -f "$(WEED_BINARY)" ]; then \ + echo "$(RED)❌ SeaweedFS binary not found at $(WEED_BINARY)$(RESET)"; \ + echo " Please run 'make' in the root directory first"; \ + exit 1; \ + fi + @echo "$(GREEN)[OK] SeaweedFS binary found at $(WEED_BINARY)$(RESET)" + +check-fuse: + @if command -v fusermount >/dev/null 2>&1; then \ + echo "$(GREEN)[OK] FUSE is installed (Linux)$(RESET)"; \ + elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \ + echo "$(GREEN)[OK] FUSE is available (macOS)$(RESET)"; \ + else \ + echo "$(RED)❌ FUSE not found. Please install:$(RESET)"; \ + echo " Ubuntu/Debian: sudo apt-get install fuse"; \ + echo " CentOS/RHEL: sudo yum install fuse"; \ + echo " macOS: brew install macfuse"; \ + exit 1; \ + fi + +check-go: + @go version | grep -Eq "go(1\.(2[1-9]|[3-9][0-9]+)|[2-9]\.)" || \ + (echo "$(RED)❌ Go $(GO_VERSION)+ required. Current: $$(go version)$(RESET)" && exit 1) + @echo "$(GREEN)[OK] Go version check passed$(RESET)" + +check-prereqs: check-go check-fuse check-binary + @echo "$(GREEN)[OK] All prerequisites satisfied$(RESET)" + +# Setup and initialization + + +setup-reports: + @mkdir -p $(REPORT_DIR) + @mkdir -p $(EXTERNAL_TOOLS_DIR) + +setup-external-tools: setup-reports + @echo "$(BLUE)🛠️ Setting up external POSIX test tools...$(RESET)" + @$(MAKE_SELF) setup-pjdfstest || echo "$(YELLOW)[WARNING] pjdfstest setup failed - continuing without it$(RESET)" + @$(MAKE_SELF) setup-nfstest || echo "$(YELLOW)[WARNING] nfstest setup failed - continuing without it$(RESET)" + @$(MAKE_SELF) setup-fio || echo "$(YELLOW)[WARNING] FIO setup failed - continuing without it$(RESET)" + +# External tools setup +setup-pjdfstest: + @if [ ! -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \ + echo "$(BLUE)[SETUP] Setting up pjdfstest...$(RESET)"; \ + if ! command -v autoreconf >/dev/null 2>&1; then \ + echo "$(YELLOW)[WARNING] autoreconf not found - installing build-essential...$(RESET)"; \ + if command -v apt-get >/dev/null 2>&1; then \ + sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool; \ + elif command -v yum >/dev/null 2>&1; then \ + sudo yum install -y gcc make autoconf automake libtool; \ + else \ + echo "$(YELLOW)[WARNING] Please install build tools manually$(RESET)"; \ + exit 1; \ + fi; \ + fi; \ + cd $(EXTERNAL_TOOLS_DIR) && \ + git clone https://github.com/pjd/pjdfstest.git && \ + cd pjdfstest && \ + autoreconf -ifs && \ + ./configure && \ + make; \ + else \ + echo "$(GREEN)[OK] pjdfstest already setup$(RESET)"; \ + fi + +setup-nfstest: + @echo "$(BLUE)[INSTALL] Installing nfstest...$(RESET)" + @pip3 install --user nfstest || \ + echo "$(YELLOW)[WARNING] nfstest installation failed. Install manually: pip3 install nfstest$(RESET)" + +setup-fio: + @if ! command -v fio >/dev/null 2>&1; then \ + echo "$(BLUE)[SETUP] Installing FIO...$(RESET)"; \ + if command -v apt-get >/dev/null 2>&1; then \ + sudo apt-get update && sudo apt-get install -y fio; \ + elif command -v yum >/dev/null 2>&1; then \ + sudo yum install -y fio; \ + elif command -v brew >/dev/null 2>&1; then \ + brew install fio; \ + else \ + echo "$(YELLOW)[WARNING] Please install FIO manually$(RESET)"; \ + fi; \ + else \ + echo "$(GREEN)[OK] FIO already installed$(RESET)"; \ + fi + +# Core test execution +test-posix-basic: check-prereqs setup-reports + @echo "$(CYAN)[TEST] Running basic POSIX compliance tests...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXCompliance . 2>&1 | \ + tee $(REPORT_DIR)/posix_basic_results.log + +test-posix-extended: check-prereqs setup-reports + @echo "$(CYAN)[TEST] Running extended POSIX compliance tests...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXExtended . 2>&1 | \ + tee $(REPORT_DIR)/posix_extended_results.log + +test-posix-external: check-prereqs setup-reports + @echo "$(CYAN)[TEST] Running external POSIX test suite integration...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \ + echo "$(GREEN)[OK] External tools available - running external tests$(RESET)"; \ + $(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestExternalPOSIXSuites . 2>&1 | \ + tee $(REPORT_DIR)/posix_external_results.log; \ + else \ + echo "$(YELLOW)[WARNING] External tools not available - skipping external tests$(RESET)"; \ + echo "External tests skipped - tools not available" > $(REPORT_DIR)/posix_external_results.log; \ + fi + +# Comprehensive test suites +test-posix-full: test-posix-basic test-posix-extended test-posix-external + @echo "$(GREEN)[OK] Full POSIX compliance test suite completed$(RESET)" + @$(MAKE_SELF) generate-report + +test-posix-critical: check-prereqs setup-reports + @echo "$(CYAN)[TEST] Running critical POSIX compliance tests...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout 15m -failfast=false \ + -run "TestPOSIXCompliance/(FileOperations|DirectoryOperations|PermissionTests|IOOperations)" \ + . 2>&1 | tee $(REPORT_DIR)/posix_critical_results.log + +test-posix-stress: check-prereqs setup-reports + @echo "$(CYAN)[TEST] Running POSIX stress tests...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \ + echo "$(GREEN)[OK] External tools available - running stress tests$(RESET)"; \ + $(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false \ + -run "TestExternalPOSIXSuites/CustomPOSIXTests" \ + . 2>&1 | tee $(REPORT_DIR)/posix_stress_results.log; \ + else \ + echo "$(YELLOW)[WARNING] External tools not available - skipping stress tests$(RESET)"; \ + echo "Stress tests skipped - tools not available" > $(REPORT_DIR)/posix_stress_results.log; \ + fi + +# Performance and benchmarks +benchmark-posix: check-prereqs setup-reports + @echo "$(CYAN)📈 Running POSIX performance benchmarks...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -run '^$$' -bench=. -benchmem \ + . 2>&1 | \ + tee $(REPORT_DIR)/posix_benchmark_results.log + +profile-posix: check-prereqs setup-reports + @echo "$(CYAN)[PROFILE] Running POSIX tests with profiling...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -cpuprofile $(REPORT_DIR)/posix.cpu.prof \ + -memprofile $(REPORT_DIR)/posix.mem.prof -run TestPOSIXCompliance . + @echo "$(GREEN)[PROFILE] Profiles generated:$(RESET)" + @echo " CPU: $(REPORT_DIR)/posix.cpu.prof" + @echo " Memory: $(REPORT_DIR)/posix.mem.prof" + @echo "$(BLUE)View with: go tool pprof $(REPORT_DIR)/posix.cpu.prof$(RESET)" + +# Coverage analysis +coverage-posix: check-prereqs setup-reports + @echo "$(CYAN)[COVERAGE] Running POSIX tests with coverage analysis...$(RESET)" +ifneq ($(TEST_MOUNT_POINT),) + @echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)" +endif + @$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -coverprofile=$(REPORT_DIR)/$(COVERAGE_FILE) \ + . + @go tool cover -html=$(REPORT_DIR)/$(COVERAGE_FILE) -o $(REPORT_DIR)/posix_coverage.html + @echo "$(GREEN)[COVERAGE] Coverage report generated: $(REPORT_DIR)/posix_coverage.html$(RESET)" + +# The external suites exercise whatever filesystem MOUNT_POINT resolves to; +# mkdir -p would happily point them at a plain local directory and report a +# pass the mount never earned. +require-mounted: + @mnt="$(MOUNT_POINT)"; mnt="$${mnt%/}"; \ + if ! mountpoint -q "$$mnt" 2>/dev/null && ! mount | grep -qF " $$mnt "; then \ + echo "$(RED)❌ $$mnt is not a mounted filesystem - refusing to run against the local disk$(RESET)"; \ + exit 1; \ + fi + +# External tool tests +test-pjdfstest: setup-pjdfstest setup-reports require-mounted + @echo "$(CYAN)[TEST] Running pjdfstest suite...$(RESET)" + @if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \ + mkdir -p $(MOUNT_POINT)/pjdfstest_workdir; \ + cd $(MOUNT_POINT)/pjdfstest_workdir && \ + prove -r $(CURDIR)/$(EXTERNAL_TOOLS_DIR)/pjdfstest/tests/ 2>&1 | tee $(CURDIR)/$(REPORT_DIR)/pjdfstest_results.log; \ + else \ + echo "$(RED)❌ pjdfstest not setup$(RESET)"; \ + exit 1; \ + fi + +test-nfstest-posix: setup-nfstest setup-reports require-mounted + @echo "$(CYAN)[TEST] Running nfstest_posix...$(RESET)" + @if command -v nfstest_posix >/dev/null 2>&1; then \ + mkdir -p $(MOUNT_POINT); \ + nfstest_posix --path $(MOUNT_POINT) --verbose 2>&1 | \ + tee $(REPORT_DIR)/nfstest_results.log; \ + else \ + echo "$(YELLOW)[WARNING] nfstest_posix not available$(RESET)"; \ + fi + +# FIO-based performance tests +test-fio-posix: setup-reports require-mounted + @echo "$(CYAN)[TEST] Running FIO-based POSIX I/O tests...$(RESET)" + @$(MAKE_SELF) create-fio-configs + @if command -v fio >/dev/null 2>&1; then \ + mkdir -p $(MOUNT_POINT); \ + for config in $(REPORT_DIR)/fio_*.conf; do \ + echo "$(BLUE)Running FIO config: $$config$(RESET)"; \ + fio $$config --output=$(REPORT_DIR)/$$(basename $$config .conf)_results.log; \ + done; \ + else \ + echo "$(YELLOW)[WARNING] FIO not available$(RESET)"; \ + fi + +create-fio-configs: setup-reports + @echo "$(BLUE)[CONFIG] Creating FIO test configurations...$(RESET)" + @echo "[global]" > $(REPORT_DIR)/fio_random_rw.conf + @echo "name=posix_random_rw" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "ioengine=sync" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "iodepth=1" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "rw=randrw" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "bs=4k" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "direct=0" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "size=100m" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "numjobs=4" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "runtime=30" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "time_based" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "[random_rw_test]" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_random_rw.conf + @echo "[global]" > $(REPORT_DIR)/fio_sequential.conf + @echo "name=posix_sequential" >> $(REPORT_DIR)/fio_sequential.conf + @echo "ioengine=sync" >> $(REPORT_DIR)/fio_sequential.conf + @echo "iodepth=1" >> $(REPORT_DIR)/fio_sequential.conf + @echo "bs=1m" >> $(REPORT_DIR)/fio_sequential.conf + @echo "direct=0" >> $(REPORT_DIR)/fio_sequential.conf + @echo "size=500m" >> $(REPORT_DIR)/fio_sequential.conf + @echo "runtime=60" >> $(REPORT_DIR)/fio_sequential.conf + @echo "time_based" >> $(REPORT_DIR)/fio_sequential.conf + @echo "" >> $(REPORT_DIR)/fio_sequential.conf + @echo "[seq_write]" >> $(REPORT_DIR)/fio_sequential.conf + @echo "rw=write" >> $(REPORT_DIR)/fio_sequential.conf + @echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf + @echo "" >> $(REPORT_DIR)/fio_sequential.conf + @echo "[seq_read]" >> $(REPORT_DIR)/fio_sequential.conf + @echo "rw=read" >> $(REPORT_DIR)/fio_sequential.conf + @echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf + +# Reporting and analysis +generate-report: setup-reports + @echo "$(BLUE)[REPORT] Generating comprehensive POSIX compliance report...$(RESET)" + @$(MAKE_SELF) generate-html-report + @$(MAKE_SELF) generate-text-report + @$(MAKE_SELF) generate-json-report + +generate-html-report: + @echo "$(BLUE)[REPORT] Generating HTML report...$(RESET)" + @echo "" > $(REPORT_DIR)/posix_compliance_report.html + @echo "SeaweedFS POSIX Compliance Report" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "

SeaweedFS POSIX Compliance Report

" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "

Generated on: $$(date)

" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "
" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "

Test Summary

" >> $(REPORT_DIR)/posix_compliance_report.html + @if [ -f "$(REPORT_DIR)/posix_basic_results.log" ]; then \ + TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \ + PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \ + FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \ + echo "

Basic Tests: $$PASSED passed, $$FAILED failed

" >> $(REPORT_DIR)/posix_compliance_report.html; \ + fi + @if [ -f "$(REPORT_DIR)/posix_critical_results.log" ]; then \ + TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \ + PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \ + FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \ + echo "

Critical Tests: $$PASSED passed, $$FAILED failed

" >> $(REPORT_DIR)/posix_compliance_report.html; \ + fi + @echo "
" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "

Detailed Results

" >> $(REPORT_DIR)/posix_compliance_report.html + @for logfile in $(REPORT_DIR)/*.log; do \ + if [ -f "$$logfile" ]; then \ + basename=$$(basename "$$logfile" .log); \ + echo "
" >> $(REPORT_DIR)/posix_compliance_report.html; \ + echo "

$$basename

" >> $(REPORT_DIR)/posix_compliance_report.html; \ + echo "
" >> $(REPORT_DIR)/posix_compliance_report.html; \
+			tail -50 "$$logfile" | sed 's/&/\&/g; s//\>/g' >> $(REPORT_DIR)/posix_compliance_report.html; \
+			echo "
" >> $(REPORT_DIR)/posix_compliance_report.html; \ + fi; \ + done + @echo "" >> $(REPORT_DIR)/posix_compliance_report.html + @echo "$(GREEN)[OK] HTML report generated: $(REPORT_DIR)/posix_compliance_report.html$(RESET)" + +generate-text-report: + @echo "$(BLUE)[REPORT] Generating text report...$(RESET)" + @echo "SeaweedFS POSIX Compliance Report" > $(REPORT_DIR)/posix_compliance_summary.txt + @echo "=================================" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "Generated: $$(date)" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "SeaweedFS Version: $$($(WEED_BINARY) version 2>/dev/null || echo 'Unknown')" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "Test Summary: See individual log files for detailed results" >> $(REPORT_DIR)/posix_compliance_summary.txt + @echo "$(GREEN)[OK] Text report generated: $(REPORT_DIR)/posix_compliance_summary.txt$(RESET)" + +generate-json-report: + @echo "$(BLUE)[REPORT] Generating JSON report...$(RESET)" + @echo "{" > $(REPORT_DIR)/posix_compliance_report.json + @echo " \"report_type\": \"posix_compliance\"," >> $(REPORT_DIR)/posix_compliance_report.json + @echo " \"timestamp\": \"$$(date -u +"%Y-%m-%dT%H:%M:%SZ")\"," >> $(REPORT_DIR)/posix_compliance_report.json + @echo " \"seaweedfs_version\": \"$$($(WEED_BINARY) version 2>/dev/null | head -n1 | sed 's/\\/\\\\/g; s/\"/\\\"/g' || echo 'Unknown')\"," >> $(REPORT_DIR)/posix_compliance_report.json + @echo " \"test_environment\": { \"os\": \"$$(uname -s)\", \"arch\": \"$$(uname -m)\" }," >> $(REPORT_DIR)/posix_compliance_report.json + @echo " \"test_suites\": {" >> $(REPORT_DIR)/posix_compliance_report.json + @FIRST=true; \ + for logfile in $(REPORT_DIR)/*.log; do \ + if [ -f "$$logfile" ]; then \ + basename=$$(basename "$$logfile" .log); \ + if [ "$$FIRST" = "true" ]; then \ + FIRST=false; \ + else \ + echo " ," >> $(REPORT_DIR)/posix_compliance_report.json; \ + fi; \ + PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \ + FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \ + TOTAL=$$(expr "$$PASSED" + "$$FAILED" 2>/dev/null || echo 0); \ + echo " \"$$basename\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"total_tests\": $$TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"passed\": $$PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"failed\": $$FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \ + if [ "$$TOTAL" -gt 0 ] 2>/dev/null; then \ + SUCCESS_RATE=$$(awk "BEGIN {printf \"%.2f\", ($$PASSED/$$TOTAL)*100}" 2>/dev/null || echo "0.00"); \ + else \ + SUCCESS_RATE="0.00"; \ + fi; \ + echo " \"success_rate\": \"$$SUCCESS_RATE%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " }" >> $(REPORT_DIR)/posix_compliance_report.json; \ + fi; \ + done + @echo " }," >> $(REPORT_DIR)/posix_compliance_report.json + @TOTAL_PASSED=0; TOTAL_FAILED=0; \ + for logfile in $(REPORT_DIR)/*.log; do \ + if [ -f "$$logfile" ]; then \ + PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \ + FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \ + TOTAL_PASSED=$$(expr "$$TOTAL_PASSED" + "$$PASSED" 2>/dev/null || echo "$$TOTAL_PASSED"); \ + TOTAL_FAILED=$$(expr "$$TOTAL_FAILED" + "$$FAILED" 2>/dev/null || echo "$$TOTAL_FAILED"); \ + fi; \ + done; \ + GRAND_TOTAL=$$(expr "$$TOTAL_PASSED" + "$$TOTAL_FAILED" 2>/dev/null || echo 0); \ + if [ "$$GRAND_TOTAL" -gt 0 ] 2>/dev/null; then \ + OVERALL_SUCCESS=$$(awk "BEGIN {printf \"%.2f\", ($$TOTAL_PASSED/$$GRAND_TOTAL)*100}" 2>/dev/null || echo "0.00"); \ + else \ + OVERALL_SUCCESS="0.00"; \ + fi; \ + echo " \"summary\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"total_tests\": $$GRAND_TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"total_passed\": $$TOTAL_PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"total_failed\": $$TOTAL_FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " \"overall_success_rate\": \"$$OVERALL_SUCCESS%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \ + echo " }" >> $(REPORT_DIR)/posix_compliance_report.json + @echo "}" >> $(REPORT_DIR)/posix_compliance_report.json + @echo "$(GREEN)[OK] JSON report generated: $(REPORT_DIR)/posix_compliance_report.json$(RESET)" + +# Cleanup and maintenance +clean: + @echo "$(YELLOW)🧹 Cleaning up test artifacts...$(RESET)" + @rm -rf $(REPORT_DIR) + @rm -rf /tmp/seaweedfs_*_test_* + @go clean -testcache + @echo "$(GREEN)[OK] Cleanup complete$(RESET)" + +clean-external-tools: + @echo "$(YELLOW)🧹 Cleaning up external tools...$(RESET)" + @rm -rf $(EXTERNAL_TOOLS_DIR) + +clean-all: clean clean-external-tools + +# Development and debugging +validate: + @echo "$(BLUE)[OK] Validating test files...$(RESET)" + @go build -o /dev/null ./... + @echo "$(GREEN)[OK] All test files compile successfully$(RESET)" + +fmt: + @echo "$(BLUE)🎨 Formatting Go code...$(RESET)" + @go fmt ./... + +lint: + @echo "$(BLUE)🔍 Running linter...$(RESET)" + @if command -v golangci-lint >/dev/null 2>&1; then \ + golangci-lint run; \ + else \ + echo "$(YELLOW)[WARNING] golangci-lint not found, running go vet instead$(RESET)"; \ + go vet ./...; \ + fi + +# CI/CD integration +ci-posix-tests: check-prereqs setup-external-tools + @echo "$(CYAN)🚀 Running POSIX tests for CI/CD...$(RESET)" + @$(MAKE_SELF) test-posix-critical + @$(MAKE_SELF) generate-report + + + +# Documentation and help +list-tests: + @echo "$(BLUE)📋 Available POSIX test functions:$(RESET)" + @grep -r "^func Test" *.go 2>/dev/null | sed 's/.*func \(Test[^(]*\).*/ \1/' | sort || echo "No test files found" + +test-info: + @echo "$(BLUE)[INFO] POSIX Test Information:$(RESET)" + @echo "Test files:" + @echo " - $(POSIX_BASIC_TESTS): Core POSIX compliance tests" + @echo " - $(POSIX_EXTENDED_TESTS): Extended POSIX feature tests" + @echo " - $(POSIX_EXTERNAL_TESTS): External test suite integration" + @echo "" + @echo "External tools:" + @echo " - pjdfstest: Comprehensive POSIX filesystem test suite" + @echo " - nfstest: Network filesystem POSIX API verification" + @echo " - FIO: Flexible I/O performance testing" + +help: + @echo "$(CYAN)SeaweedFS POSIX Compliance Testing$(RESET)" + @echo "==================================" + @echo "" + @echo "$(WHITE)Prerequisites:$(RESET)" + @echo " $(GREEN)make check-prereqs$(RESET) - Check all prerequisites" + @echo " $(GREEN)make setup-external-tools$(RESET) - Setup external POSIX test tools" + @echo "" + @echo "$(WHITE)Core POSIX Tests:$(RESET)" + @echo " $(GREEN)make test-posix-basic$(RESET) - Run basic POSIX compliance tests" + @echo " $(GREEN)make test-posix-extended$(RESET) - Run extended POSIX feature tests" + @echo " $(GREEN)make test-posix-external$(RESET) - Run external test suite integration" + @echo " $(GREEN)make test-posix-full$(RESET) - Run complete POSIX test suite" + @echo "" + @echo "$(WHITE)Focused Tests:$(RESET)" + @echo " $(GREEN)make test-posix-critical$(RESET) - Run critical POSIX compliance tests" + @echo " $(GREEN)make test-posix-stress$(RESET) - Run POSIX stress tests" + @echo "" + @echo "$(WHITE)Performance & Analysis:$(RESET)" + @echo " $(GREEN)make benchmark-posix$(RESET) - Run POSIX performance benchmarks" + @echo " $(GREEN)make profile-posix$(RESET) - Run tests with performance profiling" + @echo " $(GREEN)make coverage-posix$(RESET) - Run tests with coverage analysis" + @echo " $(GREEN)make test-fio-posix$(RESET) - Run FIO-based I/O performance tests" + @echo "" + @echo "$(WHITE)External Test Suites:$(RESET)" + @echo " $(GREEN)make test-pjdfstest$(RESET) - Run pjdfstest suite" + @echo " $(GREEN)make test-nfstest-posix$(RESET) - Run nfstest_posix" + @echo "" + @echo "$(WHITE)Reporting:$(RESET)" + @echo " $(GREEN)make generate-report$(RESET) - Generate comprehensive compliance report" + @echo "" + @echo "$(WHITE)Development:$(RESET)" + @echo " $(GREEN)make validate$(RESET) - Validate test file compilation" + @echo " $(GREEN)make fmt$(RESET) - Format Go code" + @echo " $(GREEN)make lint$(RESET) - Run linter" + @echo " $(GREEN)make list-tests$(RESET) - List available test functions" + @echo "" + @echo "$(WHITE)Maintenance:$(RESET)" + @echo " $(GREEN)make clean$(RESET) - Clean up test artifacts" + @echo " $(GREEN)make clean-all$(RESET) - Clean everything including external tools" + @echo "" + @echo "$(WHITE)CI/CD Integration:$(RESET)" + @echo " $(GREEN)make ci-posix-tests$(RESET) - Run POSIX tests optimized for CI/CD" + + +.PHONY: help check-prereqs check-binary check-fuse check-go setup-reports \ + setup-external-tools setup-pjdfstest setup-nfstest setup-fio \ + test-posix-basic test-posix-extended test-posix-external test-posix-full \ + test-posix-critical test-posix-stress benchmark-posix profile-posix coverage-posix \ + test-pjdfstest test-nfstest-posix test-fio-posix create-fio-configs \ + generate-report generate-html-report generate-text-report generate-json-report \ + clean clean-external-tools clean-all validate fmt lint ci-posix-tests \ + list-tests test-info diff --git a/test/fuse_integration/posix_compliance_test.go b/test/fuse_integration/posix_compliance_test.go new file mode 100644 index 0000000000..6ce17a3a40 --- /dev/null +++ b/test/fuse_integration/posix_compliance_test.go @@ -0,0 +1,713 @@ +//go:build !windows + +package fuse + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// POSIXComplianceTestSuite provides comprehensive POSIX compliance testing for FUSE mounts +type POSIXComplianceTestSuite struct { + framework *FuseTestFramework + t *testing.T +} + +// NewPOSIXComplianceTestSuite creates a new POSIX compliance test suite +func NewPOSIXComplianceTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXComplianceTestSuite { + return &POSIXComplianceTestSuite{ + framework: framework, + t: t, + } +} + +// TestPOSIXCompliance runs all POSIX compliance tests +func TestPOSIXCompliance(t *testing.T) { + config := DefaultTestConfig() + config.EnableDebug = true + config.MountOptions = []string{"-allowOthers", "-nonempty"} + + framework := NewFuseTestFramework(t, config) + defer framework.Cleanup() + require.NoError(t, framework.Setup(config)) + + suite := NewPOSIXComplianceTestSuite(t, framework) + + // Run all POSIX compliance test categories + t.Run("FileOperations", suite.TestFileOperations) + t.Run("DirectoryOperations", suite.TestDirectoryOperations) + t.Run("SymlinkOperations", suite.TestSymlinkOperations) + t.Run("PermissionTests", suite.TestPermissions) + t.Run("TimestampTests", suite.TestTimestamps) + t.Run("IOOperations", suite.TestIOOperations) + t.Run("FileDescriptorTests", suite.TestFileDescriptors) + t.Run("AtomicOperations", suite.TestAtomicOperations) + t.Run("ConcurrentAccess", suite.TestConcurrentAccess) + t.Run("ErrorHandling", suite.TestErrorHandling) +} + +// TestFileOperations tests POSIX file operation compliance +func (s *POSIXComplianceTestSuite) TestFileOperations(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("CreateFile", func(t *testing.T) { + filepath := filepath.Join(mountPoint, "test_create.txt") + + // Test file creation with O_CREAT + fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_WRONLY, 0644) + require.NoError(t, err) + require.Greater(t, fd, 0) + + err = syscall.Close(fd) + require.NoError(t, err) + + // Verify file exists + _, err = os.Stat(filepath) + require.NoError(t, err) + }) + + t.Run("CreateExclusiveFile", func(t *testing.T) { + filepath := filepath.Join(mountPoint, "test_excl.txt") + + // First creation should succeed + fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644) + require.NoError(t, err) + syscall.Close(fd) + + // Second creation should fail with EEXIST + _, err = syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644) + require.Error(t, err) + require.Equal(t, syscall.EEXIST, err) + }) + + t.Run("TruncateFile", func(t *testing.T) { + filepath := filepath.Join(mountPoint, "test_truncate.txt") + content := []byte("Hello, World! This is a test file for truncation.") + + // Create file with content + err := os.WriteFile(filepath, content, 0644) + require.NoError(t, err) + + // Truncate to 5 bytes + err = syscall.Truncate(filepath, 5) + require.NoError(t, err) + + // Verify truncation + readContent, err := os.ReadFile(filepath) + require.NoError(t, err) + require.Equal(t, []byte("Hello"), readContent) + }) + + t.Run("UnlinkFile", func(t *testing.T) { + filepath := filepath.Join(mountPoint, "test_unlink.txt") + + // Create file + err := os.WriteFile(filepath, []byte("test"), 0644) + require.NoError(t, err) + + // Unlink file + err = syscall.Unlink(filepath) + require.NoError(t, err) + + // Verify file no longer exists + _, err = os.Stat(filepath) + require.True(t, os.IsNotExist(err)) + }) +} + +// TestDirectoryOperations tests POSIX directory operation compliance +func (s *POSIXComplianceTestSuite) TestDirectoryOperations(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("CreateDirectory", func(t *testing.T) { + dirPath := filepath.Join(mountPoint, "test_mkdir") + + err := syscall.Mkdir(dirPath, 0755) + require.NoError(t, err) + + // Verify directory exists and has correct type + stat, err := os.Stat(dirPath) + require.NoError(t, err) + require.True(t, stat.IsDir()) + }) + + t.Run("RemoveDirectory", func(t *testing.T) { + dirPath := filepath.Join(mountPoint, "test_rmdir") + + // Create directory + err := os.Mkdir(dirPath, 0755) + require.NoError(t, err) + + // Remove directory + err = syscall.Rmdir(dirPath) + require.NoError(t, err) + + // Verify directory no longer exists + _, err = os.Stat(dirPath) + require.True(t, os.IsNotExist(err)) + }) + + t.Run("RemoveNonEmptyDirectory", func(t *testing.T) { + dirPath := filepath.Join(mountPoint, "test_rmdir_nonempty") + filePath := filepath.Join(dirPath, "file.txt") + + // Create directory and file + err := os.Mkdir(dirPath, 0755) + require.NoError(t, err) + err = os.WriteFile(filePath, []byte("test"), 0644) + require.NoError(t, err) + + // Attempt to remove non-empty directory should fail + err = syscall.Rmdir(dirPath) + require.Error(t, err) + require.Equal(t, syscall.ENOTEMPTY, err) + }) + + t.Run("RenameDirectory", func(t *testing.T) { + oldPath := filepath.Join(mountPoint, "old_dir") + newPath := filepath.Join(mountPoint, "new_dir") + + // Create directory + err := os.Mkdir(oldPath, 0755) + require.NoError(t, err) + + // Rename directory + err = os.Rename(oldPath, newPath) + require.NoError(t, err) + + // Verify old path doesn't exist and new path does + _, err = os.Stat(oldPath) + require.True(t, os.IsNotExist(err)) + stat, err := os.Stat(newPath) + require.NoError(t, err) + require.True(t, stat.IsDir()) + }) +} + +// TestSymlinkOperations tests POSIX symlink operation compliance +func (s *POSIXComplianceTestSuite) TestSymlinkOperations(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("CreateSymlink", func(t *testing.T) { + targetFile := filepath.Join(mountPoint, "target.txt") + linkFile := filepath.Join(mountPoint, "link.txt") + + // Create target file + err := os.WriteFile(targetFile, []byte("target content"), 0644) + require.NoError(t, err) + + // Create symlink + err = os.Symlink(targetFile, linkFile) + require.NoError(t, err) + + // Verify symlink properties + linkStat, err := os.Lstat(linkFile) + require.NoError(t, err) + require.Equal(t, os.ModeSymlink, linkStat.Mode()&os.ModeType) + + // Verify symlink content + linkTarget, err := os.Readlink(linkFile) + require.NoError(t, err) + require.Equal(t, targetFile, linkTarget) + + // Verify following symlink works + content, err := os.ReadFile(linkFile) + require.NoError(t, err) + require.Equal(t, []byte("target content"), content) + }) + + t.Run("BrokenSymlink", func(t *testing.T) { + nonexistentTarget := filepath.Join(mountPoint, "nonexistent.txt") + linkFile := filepath.Join(mountPoint, "broken_link.txt") + + // Create symlink to nonexistent file + err := os.Symlink(nonexistentTarget, linkFile) + require.NoError(t, err) + + // Lstat should work (doesn't follow symlink) + _, err = os.Lstat(linkFile) + require.NoError(t, err) + + // Stat should fail (follows symlink to nonexistent target) + _, err = os.Stat(linkFile) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + }) +} + +// TestPermissions tests POSIX permission compliance +func (s *POSIXComplianceTestSuite) TestPermissions(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("FilePermissions", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "perm_test.txt") + + // Create file with specific permissions + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0642) + require.NoError(t, err) + syscall.Close(fd) + + // Verify permissions + stat, err := os.Stat(testFile) + require.NoError(t, err) + + // Note: The final file permissions are affected by the system's umask. + // We check against the requested mode and the mode as affected by a common umask (e.g. 0022). + actualMode := stat.Mode() & os.ModePerm + expectedMode := os.FileMode(0642) + expectedModeWithUmask := os.FileMode(0640) // e.g., 0642 with umask 0002 or 0022 + + // Accept either the exact permissions or permissions as modified by umask + if actualMode != expectedMode && actualMode != expectedModeWithUmask { + t.Errorf("Expected file permissions %o or %o, but got %o", + expectedMode, expectedModeWithUmask, actualMode) + } + }) + + t.Run("ChangeFilePermissions", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "chmod_test.txt") + + // Create file + err := os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Change permissions + err = os.Chmod(testFile, 0755) + require.NoError(t, err) + + // Verify new permissions + stat, err := os.Stat(testFile) + require.NoError(t, err) + require.Equal(t, os.FileMode(0755), stat.Mode()&os.ModePerm) + }) + + t.Run("DirectoryPermissions", func(t *testing.T) { + testDir := filepath.Join(mountPoint, "perm_dir") + + // Create directory with specific permissions + err := syscall.Mkdir(testDir, 0750) + require.NoError(t, err) + + // Verify permissions + stat, err := os.Stat(testDir) + require.NoError(t, err) + require.Equal(t, os.FileMode(0750)|os.ModeDir, stat.Mode()) + }) +} + +// TestTimestamps tests POSIX timestamp compliance +func (s *POSIXComplianceTestSuite) TestTimestamps(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("AccessTime", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "atime_test.txt") + + // Create file + err := os.WriteFile(testFile, []byte("test content"), 0644) + require.NoError(t, err) + + // Get initial timestamps + stat1, err := os.Stat(testFile) + require.NoError(t, err) + + // Wait a bit to ensure time difference + time.Sleep(100 * time.Millisecond) + + // Read file (should update access time) + _, err = os.ReadFile(testFile) + require.NoError(t, err) + + // Get new timestamps + stat2, err := os.Stat(testFile) + require.NoError(t, err) + + // Access time should have been updated, and modification time should be unchanged. + stat1Sys := stat1.Sys().(*syscall.Stat_t) + stat2Sys := stat2.Sys().(*syscall.Stat_t) + require.True(t, getAtimeNano(stat2Sys) >= getAtimeNano(stat1Sys), "access time should be updated or stay the same") + require.Equal(t, stat1.ModTime(), stat2.ModTime(), "modification time should not change on read") + }) + + t.Run("ModificationTime", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "mtime_test.txt") + + // Create file + err := os.WriteFile(testFile, []byte("initial content"), 0644) + require.NoError(t, err) + + // Get initial timestamp + stat1, err := os.Stat(testFile) + require.NoError(t, err) + + // Wait to ensure time difference + time.Sleep(100 * time.Millisecond) + + // Modify file + err = os.WriteFile(testFile, []byte("modified content"), 0644) + require.NoError(t, err) + + // Get new timestamp + stat2, err := os.Stat(testFile) + require.NoError(t, err) + + // Modification time should have changed + require.True(t, stat2.ModTime().After(stat1.ModTime())) + }) + + t.Run("SetTimestamps", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "utime_test.txt") + + // Create file + err := os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Set specific timestamps + atime := time.Now().Add(-24 * time.Hour) + mtime := time.Now().Add(-12 * time.Hour) + + err = os.Chtimes(testFile, atime, mtime) + require.NoError(t, err) + + // Verify timestamps were set + stat, err := os.Stat(testFile) + require.NoError(t, err) + require.True(t, stat.ModTime().Equal(mtime)) + }) +} + +// TestIOOperations tests POSIX I/O operation compliance +func (s *POSIXComplianceTestSuite) TestIOOperations(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("ReadWrite", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "rw_test.txt") + testData := []byte("Hello, POSIX World!") + + // Write data + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0644) + require.NoError(t, err) + + n, err := syscall.Write(fd, testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + + err = syscall.Close(fd) + require.NoError(t, err) + + // Read data back + fd, err = syscall.Open(testFile, syscall.O_RDONLY, 0) + require.NoError(t, err) + + readBuffer := make([]byte, len(testData)) + n, err = syscall.Read(fd, readBuffer) + require.NoError(t, err) + require.Equal(t, len(testData), n) + require.Equal(t, testData, readBuffer) + + err = syscall.Close(fd) + require.NoError(t, err) + }) + + t.Run("Seek", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "seek_test.txt") + testData := []byte("0123456789ABCDEF") + + // Create file with test data + err := os.WriteFile(testFile, testData, 0644) + require.NoError(t, err) + + // Open for reading + fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0) + require.NoError(t, err) + defer syscall.Close(fd) + + // Seek to position 5 + pos, err := syscall.Seek(fd, 5, 0) // SEEK_SET + require.NoError(t, err) + require.Equal(t, int64(5), pos) + + // Read 3 bytes + buffer := make([]byte, 3) + n, err := syscall.Read(fd, buffer) + require.NoError(t, err) + require.Equal(t, 3, n) + require.Equal(t, []byte("567"), buffer) + + // Seek from current position + pos, err = syscall.Seek(fd, 2, 1) // SEEK_CUR + require.NoError(t, err) + require.Equal(t, int64(10), pos) + + // Read 1 byte + buffer = make([]byte, 1) + n, err = syscall.Read(fd, buffer) + require.NoError(t, err) + require.Equal(t, 1, n) + require.Equal(t, []byte("A"), buffer) + + // Test positioned I/O operations (pread/pwrite) + syscall.Close(fd) + + // Open for read/write to test pwrite + fd, err = syscall.Open(testFile, syscall.O_RDWR, 0) + require.NoError(t, err) + defer syscall.Close(fd) + + // Positioned write test + writeData := []byte("XYZ") + n, err = syscall.Pwrite(fd, writeData, 5) // pwrite at offset 5 + require.NoError(t, err) + require.Equal(t, len(writeData), n) + + // Verify file position is unchanged by pwrite + currentPos, err := syscall.Seek(fd, 0, 1) // SEEK_CUR + require.NoError(t, err) + require.Equal(t, int64(0), currentPos, "file offset should not be changed by pwrite") + + // Read back with pread + readBuffer := make([]byte, len(writeData)) + n, err = syscall.Pread(fd, readBuffer, 5) // pread at offset 5 + require.NoError(t, err) + require.Equal(t, len(writeData), n) + require.Equal(t, writeData, readBuffer) + + // Verify file position is still unchanged by pread + currentPos, err = syscall.Seek(fd, 0, 1) // SEEK_CUR + require.NoError(t, err) + require.Equal(t, int64(0), currentPos, "file offset should not be changed by pread") + }) + + t.Run("AppendMode", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "append_test.txt") + + // Create file with initial content + err := os.WriteFile(testFile, []byte("initial"), 0644) + require.NoError(t, err) + + // Open in append mode + fd, err := syscall.Open(testFile, syscall.O_WRONLY|syscall.O_APPEND, 0) + require.NoError(t, err) + + // Write additional content + appendData := []byte(" appended") + n, err := syscall.Write(fd, appendData) + require.NoError(t, err) + require.Equal(t, len(appendData), n) + + err = syscall.Close(fd) + require.NoError(t, err) + + // Verify content + content, err := os.ReadFile(testFile) + require.NoError(t, err) + require.Equal(t, []byte("initial appended"), content) + }) +} + +// TestFileDescriptors tests POSIX file descriptor behavior +func (s *POSIXComplianceTestSuite) TestFileDescriptors(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("DuplicateFileDescriptors", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "dup_test.txt") + testData := []byte("duplicate test") + + // Create and open file + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + // Write initial data + n, err := syscall.Write(fd, testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + + // Duplicate file descriptor + dupFd, err := syscall.Dup(fd) + require.NoError(t, err) + defer syscall.Close(dupFd) + + // Both descriptors should refer to the same file + // Seek on one should affect the other + pos, err := syscall.Seek(fd, 0, 0) // SEEK_SET + require.NoError(t, err) + require.Equal(t, int64(0), pos) + + // Read from duplicate descriptor should start from position 0 + buffer := make([]byte, 9) + n, err = syscall.Read(dupFd, buffer) + require.NoError(t, err) + require.Equal(t, 9, n) + require.Equal(t, []byte("duplicate"), buffer) + }) + + t.Run("FileDescriptorFlags", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "flags_test.txt") + + // Create file + err := os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Open with close-on-exec flag + fd, err := syscall.Open(testFile, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) + require.NoError(t, err) + defer syscall.Close(fd) + + // Verify close-on-exec flag is set + // Note: FcntlInt is not available on all platforms, this test may need platform-specific implementation + t.Skip("FcntlInt not available on this platform") + }) +} + +// TestAtomicOperations tests POSIX atomic operation compliance +func (s *POSIXComplianceTestSuite) TestAtomicOperations(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("AtomicRename", func(t *testing.T) { + oldFile := filepath.Join(mountPoint, "atomic_old.txt") + newFile := filepath.Join(mountPoint, "atomic_new.txt") + testData := []byte("atomic test data") + + // Create source file + err := os.WriteFile(oldFile, testData, 0644) + require.NoError(t, err) + + // Create existing target file + err = os.WriteFile(newFile, []byte("old content"), 0644) + require.NoError(t, err) + + // Atomic rename should replace target + err = os.Rename(oldFile, newFile) + require.NoError(t, err) + + // Verify source no longer exists + _, err = os.Stat(oldFile) + require.True(t, os.IsNotExist(err)) + + // Verify target has new content + content, err := os.ReadFile(newFile) + require.NoError(t, err) + require.Equal(t, testData, content) + }) +} + +// TestConcurrentAccess tests concurrent access patterns +func (s *POSIXComplianceTestSuite) TestConcurrentAccess(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("ConcurrentReads", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "concurrent_read.txt") + testData := []byte("concurrent read test data") + + // Create test file + err := os.WriteFile(testFile, testData, 0644) + require.NoError(t, err) + + // Launch multiple concurrent readers + const numReaders = 10 + results := make(chan error, numReaders) + + for i := 0; i < numReaders; i++ { + go func() { + content, err := os.ReadFile(testFile) + if err != nil { + results <- err + return + } + if string(content) != string(testData) { + results <- fmt.Errorf("content mismatch") + return + } + results <- nil + }() + } + + // Check all readers succeeded + for i := 0; i < numReaders; i++ { + err := <-results + require.NoError(t, err) + } + }) + + t.Run("ConcurrentFileCreations", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "concurrent_write.txt") + + // Launch multiple concurrent writers + const numWriters = 5 + results := make(chan error, numWriters) + + for i := 0; i < numWriters; i++ { + go func(id int) { + content := fmt.Sprintf("writer %d data", id) + err := os.WriteFile(fmt.Sprintf("%s_%d", testFile, id), []byte(content), 0644) + results <- err + }(i) + } + + // Check all writers succeeded + for i := 0; i < numWriters; i++ { + err := <-results + require.NoError(t, err) + } + + // Verify all files were created + for i := 0; i < numWriters; i++ { + fileName := fmt.Sprintf("%s_%d", testFile, i) + _, err := os.Stat(fileName) + require.NoError(t, err) + } + }) +} + +// TestErrorHandling tests POSIX error handling compliance +func (s *POSIXComplianceTestSuite) TestErrorHandling(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("AccessNonexistentFile", func(t *testing.T) { + nonexistentFile := filepath.Join(mountPoint, "does_not_exist.txt") + + // Reading nonexistent file should return ENOENT + _, err := os.ReadFile(nonexistentFile) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + }) + + t.Run("CreateFileInNonexistentDirectory", func(t *testing.T) { + fileInNonexistentDir := filepath.Join(mountPoint, "nonexistent_dir", "file.txt") + + // Creating file in nonexistent directory should fail + err := os.WriteFile(fileInNonexistentDir, []byte("test"), 0644) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + }) + + t.Run("InvalidFileDescriptor", func(t *testing.T) { + // Using an invalid file descriptor should return appropriate error + buffer := make([]byte, 10) + _, err := syscall.Read(999, buffer) // 999 is likely an invalid fd + require.Error(t, err) + require.Equal(t, syscall.EBADF, err) + }) + + t.Run("PermissionDenied", func(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: mode bits are not enforced") + } + testFile := filepath.Join(mountPoint, "readonly.txt") + + // Create read-only file + err := os.WriteFile(testFile, []byte("readonly"), 0444) + require.NoError(t, err) + + // Attempting to write to read-only file should fail + err = os.WriteFile(testFile, []byte("modified"), 0444) + require.Error(t, err) + require.True(t, os.IsPermission(err)) + }) +} diff --git a/test/fuse_integration/posix_extended_test.go b/test/fuse_integration/posix_extended_test.go new file mode 100644 index 0000000000..7e8cbd7724 --- /dev/null +++ b/test/fuse_integration/posix_extended_test.go @@ -0,0 +1,758 @@ +//go:build !windows + +package fuse + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// POSIXExtendedTestSuite provides additional POSIX compliance tests +// covering extended attributes, file locking, and advanced features. +// +// NOTE: Some tests in this suite may be skipped or may have different behavior +// depending on the FUSE implementation and platform support. This is expected +// behavior for comprehensive POSIX compliance testing. +// +// See POSIX_IMPLEMENTATION_ROADMAP.md for a comprehensive plan to implement +// these missing features. +type POSIXExtendedTestSuite struct { + framework *FuseTestFramework + t *testing.T +} + +// ErrorCollector helps collect multiple test errors without stopping execution +type ErrorCollector struct { + errors []string + t *testing.T +} + +// NewErrorCollector creates a new error collector +func NewErrorCollector(t *testing.T) *ErrorCollector { + return &ErrorCollector{ + errors: make([]string, 0), + t: t, + } +} + +// Add adds an error to the collection +func (ec *ErrorCollector) Add(format string, args ...interface{}) { + ec.errors = append(ec.errors, fmt.Sprintf(format, args...)) +} + +// Check checks a condition and adds an error if it fails +func (ec *ErrorCollector) Check(condition bool, format string, args ...interface{}) { + if !condition { + ec.Add(format, args...) + } +} + +// CheckError checks if an error is nil and adds it if not +func (ec *ErrorCollector) CheckError(err error, format string, args ...interface{}) { + if err != nil { + ec.Add(format+": %v", append(args, err)...) + } +} + +// HasErrors returns true if any errors were collected +func (ec *ErrorCollector) HasErrors() bool { + return len(ec.errors) > 0 +} + +// Report reports all collected errors to the test +func (ec *ErrorCollector) Report() { + if len(ec.errors) > 0 { + ec.t.Errorf("Test completed with %d errors:\n%s", len(ec.errors), strings.Join(ec.errors, "\n")) + } +} + +// NewPOSIXExtendedTestSuite creates a new extended POSIX compliance test suite +func NewPOSIXExtendedTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXExtendedTestSuite { + return &POSIXExtendedTestSuite{ + framework: framework, + t: t, + } +} + +// TestPOSIXExtended runs extended POSIX compliance tests +func TestPOSIXExtended(t *testing.T) { + config := DefaultTestConfig() + config.EnableDebug = true + config.MountOptions = []string{"-allowOthers", "-nonempty"} + + framework := NewFuseTestFramework(t, config) + defer framework.Cleanup() + + // Setup framework with better error handling + err := framework.Setup(config) + if err != nil { + t.Logf("Framework setup failed: %v", err) + t.Skip("Skipping extended tests due to framework setup failure") + return + } + + // Verify framework is working + mountPoint := framework.GetMountPoint() + if mountPoint == "" { + t.Log("Framework mount point is empty") + t.Skip("Skipping extended tests due to invalid mount point") + return + } + + suite := NewPOSIXExtendedTestSuite(t, framework) + + // Run extended POSIX compliance test categories + // Use a resilient approach that continues even if individual tests fail + testCategories := []struct { + name string + fn func(*testing.T) + }{ + {"ExtendedAttributes", suite.TestExtendedAttributes}, + {"FileLocking", suite.TestFileLocking}, + {"AdvancedIO", suite.TestAdvancedIO}, + {"SparseFiles", suite.TestSparseFiles}, + {"LargeFiles", suite.TestLargeFiles}, + {"MMap", suite.TestMemoryMapping}, + {"DirectIO", suite.TestDirectIO}, + {"FileSealing", suite.TestFileSealing}, + {"Fallocate", suite.TestFallocate}, + {"Sendfile", suite.TestSendfile}, + } + + // Run all test categories and collect results + var failedCategories []string + for _, category := range testCategories { + t.Run(category.name, func(subT *testing.T) { + // Capture panics and convert them to test failures + defer func() { + if r := recover(); r != nil { + subT.Errorf("Test category %s panicked: %v", category.name, r) + failedCategories = append(failedCategories, category.name) + } + }() + + category.fn(subT) + if subT.Failed() { + failedCategories = append(failedCategories, category.name) + } + }) + } + + // Report overall results + if len(failedCategories) > 0 { + t.Logf("Extended POSIX tests completed. Failed categories: %v", failedCategories) + } else { + t.Logf("All extended POSIX test categories passed!") + } +} + +// TestExtendedAttributes tests POSIX extended attribute support +func (s *POSIXExtendedTestSuite) TestExtendedAttributes(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("SetAndGetXattr", func(t *testing.T) { + if !isXattrSupported() { + t.Skip("Extended attributes not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "xattr_test.txt") + + // Create test file + err := os.WriteFile(testFile, []byte("xattr test"), 0644) + if !assert.NoError(t, err, "Failed to create test file") { + return + } + + // Set extended attribute + attrName := "user.test_attr" + attrValue := []byte("test_value") + err = setXattr(testFile, attrName, attrValue, 0) + assert.NoError(t, err, "Failed to set extended attribute") + + // Verify attribute was set + readValue := make([]byte, 256) + size, err := getXattr(testFile, attrName, readValue) + assert.NoError(t, err, "Failed to get extended attribute") + assert.Equal(t, len(attrValue), size, "Extended attribute size mismatch") + assert.Equal(t, attrValue, readValue[:size], "Extended attribute value mismatch") + }) + + t.Run("ListXattrs", func(t *testing.T) { + if !isXattrSupported() { + t.Skip("Extended attributes not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "xattr_list_test.txt") + + // Create test file + err := os.WriteFile(testFile, []byte("list xattr test"), 0644) + if !assert.NoError(t, err, "Failed to create test file") { + return + } + + // Set multiple extended attributes + attrs := map[string][]byte{ + "user.attr1": []byte("value1"), + "user.attr2": []byte("value2"), + "user.attr3": []byte("value3"), + } + + for name, value := range attrs { + err = setXattr(testFile, name, value, 0) + assert.NoError(t, err, "Failed to set extended attribute %s", name) + } + + // List all attributes + listBuf := make([]byte, 1024) + size, err := listXattr(testFile, listBuf) + assert.NoError(t, err, "Failed to list extended attributes") + assert.Greater(t, size, 0, "No extended attributes found") + + // Parse the null-separated list and verify attributes + if size > 0 { + attrList := parseXattrList(listBuf[:size]) + expectedAttrs := []string{"user.attr1", "user.attr2", "user.attr3"} + for _, expectedAttr := range expectedAttrs { + assert.Contains(t, attrList, expectedAttr, "Expected attribute %s should be in the list", expectedAttr) + } + } + }) + + t.Run("RemoveXattr", func(t *testing.T) { + if !isXattrSupported() { + t.Skip("Extended attributes not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "xattr_remove_test.txt") + + // Create test file + err := os.WriteFile(testFile, []byte("remove xattr test"), 0644) + require.NoError(t, err) + + // Set extended attribute + attrName := "user.remove_test" + attrValue := []byte("to_be_removed") + err = setXattr(testFile, attrName, attrValue, 0) + require.NoError(t, err) + + // Verify attribute exists + readValue := make([]byte, 256) + size, err := getXattr(testFile, attrName, readValue) + require.NoError(t, err) + require.Equal(t, len(attrValue), size) + + // Remove the attribute + err = removeXattr(testFile, attrName) + require.NoError(t, err) + + // Verify attribute is gone + _, err = getXattr(testFile, attrName, readValue) + require.Error(t, err) // Should fail with ENODATA or similar + }) +} + +// TestFileLocking tests POSIX file locking mechanisms +// Note: File locking behavior may vary between FUSE implementations. +// Some implementations may not enforce locks between file descriptors +// from the same process, which is a known limitation. +func (s *POSIXExtendedTestSuite) TestFileLocking(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("AdvisoryLocking", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "lock_test.txt") + + // Create test file + err := os.WriteFile(testFile, []byte("locking test"), 0644) + if !assert.NoError(t, err, "Failed to create test file") { + return + } + + // Open file + file, err := os.OpenFile(testFile, os.O_RDWR, 0644) + if !assert.NoError(t, err, "Failed to open test file") { + return + } + defer file.Close() + + // Apply exclusive lock + flock := syscall.Flock_t{ + Type: syscall.F_WRLCK, + Whence: 0, + Start: 0, + Len: 0, // Lock entire file + } + + err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock) + assert.NoError(t, err, "Failed to acquire exclusive lock") + + // Try to lock from another file descriptor (should fail) + file2, err := os.OpenFile(testFile, os.O_RDWR, 0644) + if !assert.NoError(t, err, "Failed to open second file descriptor") { + return + } + defer file2.Close() + + flock2 := syscall.Flock_t{ + Type: syscall.F_WRLCK, + Whence: 0, + Start: 0, + Len: 0, + } + + // Note: Some FUSE implementations may not enforce file locking between file descriptors + // from the same process. This is a known limitation in some FUSE implementations. + err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2) + if err == syscall.EAGAIN { + // Lock was properly enforced + t.Log("File locking properly enforced between file descriptors") + } else if err == nil { + // Lock was not enforced (common in some FUSE implementations) + t.Log("File locking not enforced between file descriptors from same process (FUSE limitation)") + } else { + // Some other error occurred + t.Logf("File locking attempt resulted in error: %v", err) + } + + // Log the actual error for debugging + t.Logf("File locking test completed with result: %v", err) + + // Release lock + flock.Type = syscall.F_UNLCK + err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock) + assert.NoError(t, err, "Failed to release lock") + + // Now second lock should succeed + err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2) + assert.NoError(t, err, "Failed to acquire lock after release") + }) + + t.Run("SharedLocking", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "shared_lock_test.txt") + + // Create test file + err := os.WriteFile(testFile, []byte("shared locking test"), 0644) + if !assert.NoError(t, err, "Failed to create test file") { + return + } + + // Open file for reading + file1, err := os.Open(testFile) + if !assert.NoError(t, err, "Failed to open first file descriptor") { + return + } + defer file1.Close() + + file2, err := os.Open(testFile) + if !assert.NoError(t, err, "Failed to open second file descriptor") { + return + } + defer file2.Close() + + // Apply shared locks (should both succeed) + flock1 := syscall.Flock_t{ + Type: syscall.F_RDLCK, + Whence: 0, + Start: 0, + Len: 0, + } + + flock2 := syscall.Flock_t{ + Type: syscall.F_RDLCK, + Whence: 0, + Start: 0, + Len: 0, + } + + err = syscall.FcntlFlock(file1.Fd(), syscall.F_SETLK, &flock1) + assert.NoError(t, err, "Failed to acquire first shared lock") + + err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2) + assert.NoError(t, err, "Failed to acquire second shared lock") + }) +} + +// TestAdvancedIO tests advanced I/O operations +// Note: Vectored I/O support may vary between FUSE implementations. +// Some implementations may not properly support readv/writev operations. +func (s *POSIXExtendedTestSuite) TestAdvancedIO(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("ReadWriteV", func(t *testing.T) { + if !isVectoredIOSupported() { + t.Skip("Vectored I/O not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "readwritev_test.txt") + + // Create file + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + if !assert.NoError(t, err, "Failed to create test file") { + return + } + defer syscall.Close(fd) + + // Prepare test data in multiple buffers + writeBuffers := [][]byte{ + []byte("Hello "), + []byte("vectored "), + []byte("I/O "), + []byte("world!"), + } + + // Write using writev + writeIOVs := makeIOVecs(writeBuffers) + t.Logf("Attempting vectored I/O write with %d buffers", len(writeIOVs)) + totalWritten, err := writevFile(fd, writeIOVs) + if err != nil { + // Some FUSE implementations may not support vectored I/O properly + t.Logf("Vectored I/O write failed: %v - this may be a FUSE limitation", err) + t.Skip("Vectored I/O not properly supported by this FUSE implementation") + return + } + + expectedTotal := 0 + for _, buf := range writeBuffers { + expectedTotal += len(buf) + } + assert.Equal(t, expectedTotal, totalWritten, "Vectored write size mismatch") + + // Seek back to beginning + _, err = syscall.Seek(fd, 0, 0) + if !assert.NoError(t, err, "Failed to seek to beginning") { + return + } + + // Read using readv into multiple buffers + readBuffers := [][]byte{ + make([]byte, 6), // "Hello " + make([]byte, 9), // "vectored " + make([]byte, 4), // "I/O " + make([]byte, 6), // "world!" + } + + readIOVs := makeIOVecs(readBuffers) + t.Logf("Attempting vectored I/O read with %d buffers", len(readIOVs)) + totalRead, err := readvFile(fd, readIOVs) + if err != nil { + t.Logf("Vectored I/O read failed: %v - this may be a FUSE limitation", err) + t.Skip("Vectored I/O not properly supported by this FUSE implementation") + return + } + assert.Equal(t, expectedTotal, totalRead, "Vectored read size mismatch") + + // Verify data matches + for i, expected := range writeBuffers { + assert.Equal(t, expected, readBuffers[i], "Vectored I/O data mismatch at buffer %d", i) + } + }) + +} + +// TestSparseFiles tests sparse file handling +func (s *POSIXExtendedTestSuite) TestSparseFiles(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("CreateSparseFile", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "sparse_test.txt") + + // Open file + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + // Create a sparse file by seeking beyond end and writing + const sparseSize = 1024 * 1024 // 1MB + _, err = syscall.Seek(fd, sparseSize, 0) + require.NoError(t, err) + + // Write at the end + endData := []byte("end") + n, err := syscall.Write(fd, endData) + require.NoError(t, err) + require.Equal(t, len(endData), n) + + // Verify file size + stat, err := os.Stat(testFile) + require.NoError(t, err) + require.Equal(t, int64(sparseSize+len(endData)), stat.Size()) + + // Read from the beginning (should be zeros) + _, err = syscall.Seek(fd, 0, 0) + require.NoError(t, err) + + buffer := make([]byte, 100) + n, err = syscall.Read(fd, buffer) + require.NoError(t, err) + require.Equal(t, 100, n) + + // Should be all zeros + for _, b := range buffer { + require.Equal(t, byte(0), b) + } + }) +} + +// TestLargeFiles tests large file handling (>2GB) +func (s *POSIXExtendedTestSuite) TestLargeFiles(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("LargeFileOperations", func(t *testing.T) { + testFile := filepath.Join(mountPoint, "large_file_test.txt") + + // Open file + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + // Seek to position > 2GB + const largeOffset = 3 * 1024 * 1024 * 1024 // 3GB + pos, err := syscall.Seek(fd, largeOffset, 0) + require.NoError(t, err) + require.Equal(t, int64(largeOffset), pos) + + // Write at large offset + testData := []byte("large file data") + n, err := syscall.Write(fd, testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + + // Read back from large offset + _, err = syscall.Seek(fd, largeOffset, 0) + require.NoError(t, err) + + readBuffer := make([]byte, len(testData)) + n, err = syscall.Read(fd, readBuffer) + require.NoError(t, err) + require.Equal(t, len(testData), n) + require.Equal(t, testData, readBuffer) + }) +} + +// TestMemoryMapping tests memory mapping functionality +func (s *POSIXExtendedTestSuite) TestMemoryMapping(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("MmapFile", func(t *testing.T) { + if !isMmapSupported() { + t.Skip("Memory mapping not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "mmap_test.txt") + testData := make([]byte, 4096) + for i := range testData { + testData[i] = byte(i % 256) + } + + // Create test file + err := os.WriteFile(testFile, testData, 0644) + require.NoError(t, err) + + // Open file for reading + fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0) + require.NoError(t, err) + defer syscall.Close(fd) + + // Memory map the file + mappedData, err := mmapFile(fd, 0, len(testData), PROT_READ, MAP_SHARED) + require.NoError(t, err) + defer munmapFile(mappedData) + + // Verify mapped content matches original + require.Equal(t, testData, mappedData) + }) + + t.Run("MmapWrite", func(t *testing.T) { + if !isMmapSupported() { + t.Skip("Memory mapping not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "mmap_write_test.txt") + size := 4096 + + // Create empty file of specific size + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + err = syscall.Ftruncate(fd, int64(size)) + require.NoError(t, err) + + // Memory map the file for writing + mappedData, err := mmapFile(fd, 0, size, PROT_READ|PROT_WRITE, MAP_SHARED) + require.NoError(t, err) + defer munmapFile(mappedData) + + // Write test pattern to mapped memory + testPattern := []byte("Hello, mmap world!") + copy(mappedData, testPattern) + + // Sync changes to disk + err = msyncFile(mappedData, MS_SYNC) + require.NoError(t, err) + + // Verify changes were written by reading file directly + readData, err := os.ReadFile(testFile) + require.NoError(t, err) + require.True(t, len(readData) >= len(testPattern)) + require.Equal(t, testPattern, readData[:len(testPattern)]) + }) +} + +// TestDirectIO tests direct I/O operations +func (s *POSIXExtendedTestSuite) TestDirectIO(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("DirectIO", func(t *testing.T) { + if !isDirectIOSupported() { + t.Skip("Direct I/O not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "directio_test.txt") + + // Open file with direct I/O + fd, err := openDirectIO(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + // For direct I/O, data must be aligned to sector boundaries + // Use 4KB aligned buffer (common sector size) + blockSize := 4096 + testData := make([]byte, blockSize) + for i := range testData { + testData[i] = byte(i % 256) + } + + // Write data using direct I/O + n, err := syscall.Write(fd, testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + + // Seek back to beginning + _, err = syscall.Seek(fd, 0, 0) + require.NoError(t, err) + + // Read data using direct I/O + readData := make([]byte, blockSize) + n, err = syscall.Read(fd, readData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + require.Equal(t, testData, readData) + }) +} + +// TestFileSealing tests file sealing mechanisms (Linux-specific) +func (s *POSIXExtendedTestSuite) TestFileSealing(t *testing.T) { + // This test is Linux-specific and may not be applicable to all systems + t.Skip("File sealing tests require Linux-specific features") +} + +// TestFallocate tests file preallocation +func (s *POSIXExtendedTestSuite) TestFallocate(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("FallocateSpace", func(t *testing.T) { + if !isFallocateSupported() { + t.Skip("fallocate not supported on this platform") + return + } + + testFile := filepath.Join(mountPoint, "fallocate_test.txt") + + // Open file + fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644) + require.NoError(t, err) + defer syscall.Close(fd) + + // Preallocate 1MB of space + allocSize := int64(1024 * 1024) + err = fallocateFile(fd, 0, 0, allocSize) + require.NoError(t, err) + + // Verify file size was extended + var stat syscall.Stat_t + err = syscall.Fstat(fd, &stat) + require.NoError(t, err) + require.GreaterOrEqual(t, stat.Size, allocSize) + + // Write some data and verify it works + testData := []byte("fallocate test data") + n, err := syscall.Write(fd, testData) + require.NoError(t, err) + require.Equal(t, len(testData), n) + }) +} + +// TestSendfile tests zero-copy file transfer +func (s *POSIXExtendedTestSuite) TestSendfile(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + t.Run("SendfileCopy", func(t *testing.T) { + sourceFile := filepath.Join(mountPoint, "sendfile_source.txt") + targetFile := filepath.Join(mountPoint, "sendfile_target.txt") + + testData := make([]byte, 8192) + for i := range testData { + testData[i] = byte(i % 256) + } + + // Create source file + err := os.WriteFile(sourceFile, testData, 0644) + require.NoError(t, err) + + // Open source for reading + srcFd, err := syscall.Open(sourceFile, syscall.O_RDONLY, 0) + require.NoError(t, err) + defer syscall.Close(srcFd) + + // Create target file + dstFd, err := syscall.Open(targetFile, syscall.O_CREAT|syscall.O_WRONLY, 0644) + require.NoError(t, err) + defer syscall.Close(dstFd) + + // Sendfile test + if !isSendfileSupported() { + t.Skip("sendfile not supported on this platform") + return + } + + // Use sendfile to copy data + var offset int64 = 0 + transferred, err := sendfileTransfer(dstFd, srcFd, &offset, len(testData)) + require.NoError(t, err) + require.Equal(t, len(testData), transferred) + + // Verify copy + copiedData, err := os.ReadFile(targetFile) + require.NoError(t, err) + require.Equal(t, testData, copiedData) + }) +} + +// Helper function to parse null-separated xattr list +func parseXattrList(data []byte) []string { + var attrs []string + start := 0 + for i, b := range data { + if b == 0 { + if i > start { + attrs = append(attrs, string(data[start:i])) + } + start = i + 1 + } + } + return attrs +} diff --git a/test/fuse_integration/posix_external_test.go b/test/fuse_integration/posix_external_test.go new file mode 100644 index 0000000000..2c63fcb929 --- /dev/null +++ b/test/fuse_integration/posix_external_test.go @@ -0,0 +1,576 @@ +package fuse + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ExternalPOSIXTestSuite manages integration with external POSIX test suites +type ExternalPOSIXTestSuite struct { + framework *FuseTestFramework + t *testing.T + workDir string +} + +// NewExternalPOSIXTestSuite creates a new external POSIX test suite runner +func NewExternalPOSIXTestSuite(t *testing.T, framework *FuseTestFramework) *ExternalPOSIXTestSuite { + workDir, err := os.MkdirTemp("", "posix_external_tests_") + require.NoError(t, err) + + return &ExternalPOSIXTestSuite{ + framework: framework, + t: t, + workDir: workDir, + } +} + +// Cleanup removes temporary test directories +func (s *ExternalPOSIXTestSuite) Cleanup() { + os.RemoveAll(s.workDir) +} + +// TestExternalPOSIXSuites runs integration tests with external POSIX test suites +func TestExternalPOSIXSuites(t *testing.T) { + config := DefaultTestConfig() + config.EnableDebug = true + config.MountOptions = []string{"-allowOthers", "-nonempty"} + + framework := NewFuseTestFramework(t, config) + defer framework.Cleanup() + require.NoError(t, framework.Setup(config)) + + suite := NewExternalPOSIXTestSuite(t, framework) + defer suite.Cleanup() + + // Run various external POSIX test suites + t.Run("PjdFsTest", suite.TestPjdFsTest) + t.Run("NFSTestPOSIX", suite.TestNFSTestPOSIX) + t.Run("LitmusTests", suite.TestLitmusTests) + t.Run("CustomPOSIXTests", suite.TestCustomPOSIXTests) +} + +// TestPjdFsTest runs the comprehensive pjdfstest POSIX filesystem test suite +func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + // Create test directory within mount point + testDir := filepath.Join(mountPoint, "pjdfstest") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + // List of critical POSIX operations to test + pjdTests := []struct { + name string + testPath string + critical bool + }{ + {"chflags", "tests/chflags", false}, + {"chmod", "tests/chmod", true}, + {"chown", "tests/chown", true}, + {"create", "tests/create", true}, + {"link", "tests/link", true}, + {"mkdir", "tests/mkdir", true}, + {"mkfifo", "tests/mkfifo", false}, + {"mknod", "tests/mknod", false}, + {"open", "tests/open", true}, + {"rename", "tests/rename", true}, + {"rmdir", "tests/rmdir", true}, + {"symlink", "tests/symlink", true}, + {"truncate", "tests/truncate", true}, + {"unlink", "tests/unlink", true}, + } + + // Prefer the suite that `make setup-pjdfstest` builds under + // external_tools/ before cloning a second copy into the work dir + pjdDir := s.findPjdDir() + if pjdDir == "" { + pjdDir = filepath.Join(s.workDir, "pjdfstest") + if err := s.setupPjdFsTest(pjdDir); err != nil { + if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" { + t.Fatalf("pjdfstest is required for comprehensive POSIX compliance testing: %v", err) + } + t.Skipf("pjdfstest not found and setup failed: %v", err) + } + } + + // Run each test category + for _, test := range pjdTests { + t.Run(test.name, func(t *testing.T) { + s.runPjdTest(t, pjdDir, test.testPath, testDir, test.critical) + }) + } +} + +// TestNFSTestPOSIX runs nfstest_posix for POSIX API verification +func (s *ExternalPOSIXTestSuite) TestNFSTestPOSIX(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + // Check if nfstest_posix is available + _, err := exec.LookPath("nfstest_posix") + if err != nil { + // For comprehensive POSIX compliance testing, external tools should be mandatory + if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" { + t.Fatalf("nfstest_posix is required for comprehensive POSIX compliance testing but not found. Install via: pip install nfstest") + } + t.Skip("nfstest_posix not found. Install via: pip install nfstest") + } + + testDir := filepath.Join(mountPoint, "nfstest") + err = os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + // Run nfstest_posix with comprehensive API testing + cmd := exec.Command("nfstest_posix", + "--path", testDir, + "--verbose", + "--createlog", + "--runid", fmt.Sprintf("seaweedfs_%d", time.Now().Unix()), + ) + + output, err := cmd.CombinedOutput() + t.Logf("nfstest_posix output:\n%s", string(output)) + + if err != nil { + t.Logf("nfstest_posix failed: %v", err) + // Don't fail the test completely, just log the failure + } +} + +// TestLitmusTests runs focused POSIX compliance litmus tests +func (s *ExternalPOSIXTestSuite) TestLitmusTests(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + // Create litmus test scripts for critical POSIX behaviors + litmusTests := []struct { + name string + script string + }{ + { + name: "AtomicRename", + script: `#!/bin/bash +set -e +cd "$1" +echo "test data" > temp_file +echo "original" > target_file +mv temp_file target_file +[ "$(cat target_file)" = "test data" ] +echo "PASS: Atomic rename works" +`, + }, + { + name: "LinkCount", + script: `#!/bin/bash +set -e +cd "$1" +echo "test" > original +ln original hardlink +if [[ "$(uname)" == "Darwin" ]]; then + [ $(stat -f %l original) -eq 2 ] +else + [ $(stat -c %h original) -eq 2 ] +fi +rm hardlink +if [[ "$(uname)" == "Darwin" ]]; then + [ $(stat -f %l original) -eq 1 ] +else + [ $(stat -c %h original) -eq 1 ] +fi +echo "PASS: Hard link counting works" +`, + }, + { + name: "SymlinkCycles", + script: `#!/bin/bash +set -e +cd "$1" +ln -s link1 link2 +ln -s link2 link1 +# Try to access the file, which should fail due to too many symbolic links +if cat link1 2>/dev/null; then + echo "FAIL: Symlink cycle not detected" + exit 1 +fi +echo "PASS: Symlink cycle handling works" +`, + }, + { + name: "ConcurrentCreate", + script: `#!/bin/bash +set -e +cd "$1" +for i in {1..10}; do + (echo "process $i" > "file_$i") & +done +wait +[ $(ls file_* | wc -l) -eq 10 ] +echo "PASS: Concurrent file creation works" +`, + }, + { + name: "DirectoryConsistency", + script: `#!/bin/bash +set -e +cd "$1" +mkdir testdir +cd testdir +touch file1 file2 file3 +cd .. +entries=$(ls testdir | wc -l) +[ $entries -eq 3 ] +rmdir testdir 2>/dev/null && echo "FAIL: Non-empty directory removed" && exit 1 +rm testdir/* +rmdir testdir +echo "PASS: Directory consistency works" +`, + }, + } + + testDir := filepath.Join(mountPoint, "litmus") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + // Run each litmus test + for _, test := range litmusTests { + t.Run(test.name, func(t *testing.T) { + s.runLitmusTest(t, test.name, test.script, testDir) + }) + } +} + +// TestCustomPOSIXTests runs custom POSIX compliance tests +func (s *ExternalPOSIXTestSuite) TestCustomPOSIXTests(t *testing.T) { + mountPoint := s.framework.GetMountPoint() + + // Custom stress tests for POSIX compliance + t.Run("StressRename", func(t *testing.T) { + s.stressTestRename(t, mountPoint) + }) + + t.Run("StressCreate", func(t *testing.T) { + s.stressTestCreate(t, mountPoint) + }) + + t.Run("StressDirectory", func(t *testing.T) { + s.stressTestDirectory(t, mountPoint) + }) + + t.Run("EdgeCases", func(t *testing.T) { + s.testEdgeCases(t, mountPoint) + }) +} + +// findPjdDir returns a directory holding a built pjdfstest suite, or "" +// when none is installed. The path is absolute because tests run with +// cmd.Dir inside the mount. +func (s *ExternalPOSIXTestSuite) findPjdDir() string { + dir := filepath.Join("external_tools", "pjdfstest") + if info, err := os.Stat(filepath.Join(dir, "pjdfstest")); err == nil && !info.IsDir() && info.Mode()&0111 != 0 { + if abs, err := filepath.Abs(dir); err == nil { + return abs + } + } + return "" +} + +// setupPjdFsTest downloads and sets up the pjdfstest suite +func (s *ExternalPOSIXTestSuite) setupPjdFsTest(targetDir string) error { + // Clone pjdfstest repository + cmd := exec.Command("git", "clone", "https://github.com/pjd/pjdfstest.git", targetDir) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to clone pjdfstest: %w", err) + } + + // pjdfstest is autotools-based: the fresh clone has no Makefile until + // configure runs + for _, step := range [][]string{ + {"autoreconf", "-ifs"}, + {"./configure"}, + {"make"}, + } { + cmd = exec.Command(step[0], step[1:]...) + cmd.Dir = targetDir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("pjdfstest %s: %w: %s", step[0], err, out) + } + } + + return nil +} + +// runPjdTest executes a specific pjdfstest +func (s *ExternalPOSIXTestSuite) runPjdTest(t *testing.T, pjdDir, testPath, mountDir string, critical bool) { + fullTestPath := filepath.Join(pjdDir, testPath) + + // Check if test path exists + if _, err := os.Stat(fullTestPath); os.IsNotExist(err) { + t.Skipf("Test path does not exist: %s", fullTestPath) + } + + cmd := exec.Command("prove", "-r", fullTestPath) + cmd.Dir = mountDir + cmd.Env = append(os.Environ(), "FSTEST="+filepath.Join(pjdDir, "pjdfstest")) + + output, err := cmd.CombinedOutput() + t.Logf("pjdfstest %s output:\n%s", testPath, string(output)) + + if err != nil { + if critical { + t.Errorf("Critical pjdfstest failed: %s - %v", testPath, err) + } else { + t.Logf("Non-critical pjdfstest failed: %s - %v", testPath, err) + } + } +} + +// runLitmusTest executes a litmus test script +func (s *ExternalPOSIXTestSuite) runLitmusTest(t *testing.T, name, script, testDir string) { + scriptPath := filepath.Join(s.workDir, name+".sh") + + // Write script to file + err := os.WriteFile(scriptPath, []byte(script), 0755) + require.NoError(t, err) + + // Create isolated test directory + isolatedDir := filepath.Join(testDir, name) + err = os.MkdirAll(isolatedDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(isolatedDir) + + // Execute script + cmd := exec.Command("bash", scriptPath, isolatedDir) + output, err := cmd.CombinedOutput() + + t.Logf("Litmus test %s output:\n%s", name, string(output)) + + if err != nil { + t.Errorf("Litmus test %s failed: %v", name, err) + } +} + +// stressTestRename tests rename operations under stress +func (s *ExternalPOSIXTestSuite) stressTestRename(t *testing.T, mountPoint string) { + testDir := filepath.Join(mountPoint, "stress_rename") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(testDir) + + // Create files and rename them concurrently + const numFiles = 100 + const numWorkers = 10 + + // Create initial files + for i := 0; i < numFiles; i++ { + fileName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i)) + err := os.WriteFile(fileName, []byte(fmt.Sprintf("content_%d", i)), 0644) + if !assert.NoError(t, err, "Failed to create initial file %d", i) { + return // Skip test if setup fails + } + } + + // Concurrent rename operations + results := make(chan error, numWorkers) + for w := 0; w < numWorkers; w++ { + go func(worker int) { + for i := worker; i < numFiles; i += numWorkers { + oldName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i)) + newName := filepath.Join(testDir, fmt.Sprintf("renamed_%d.txt", i)) + err := os.Rename(oldName, newName) + if err != nil { + results <- err + return + } + } + results <- nil + }(w) + } + + // Wait for all workers and collect errors + var errorCount int + for w := 0; w < numWorkers; w++ { + err := <-results + if err != nil { + assert.NoError(t, err, "Worker %d failed", w) + errorCount++ + } + } + + // Verify all files were renamed (if no errors occurred) + if errorCount == 0 { + files, err := filepath.Glob(filepath.Join(testDir, "renamed_*.txt")) + assert.NoError(t, err, "Failed to list renamed files") + assert.Equal(t, numFiles, len(files), "Not all files were renamed successfully") + } +} + +// stressTestCreate tests file creation under stress +func (s *ExternalPOSIXTestSuite) stressTestCreate(t *testing.T, mountPoint string) { + testDir := filepath.Join(mountPoint, "stress_create") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(testDir) + + const numFiles = 1000 + const numWorkers = 20 + + results := make(chan error, numWorkers) + + for w := 0; w < numWorkers; w++ { + go func(worker int) { + for i := worker; i < numFiles; i += numWorkers { + fileName := filepath.Join(testDir, fmt.Sprintf("stress_%d_%d.txt", worker, i)) + err := os.WriteFile(fileName, []byte(fmt.Sprintf("data_%d_%d", worker, i)), 0644) + if err != nil { + results <- err + return + } + } + results <- nil + }(w) + } + + // Wait for all workers + for w := 0; w < numWorkers; w++ { + err := <-results + require.NoError(t, err) + } + + // Verify all files were created + files, err := filepath.Glob(filepath.Join(testDir, "stress_*.txt")) + require.NoError(t, err) + require.Equal(t, numFiles, len(files)) +} + +// stressTestDirectory tests directory operations under stress +func (s *ExternalPOSIXTestSuite) stressTestDirectory(t *testing.T, mountPoint string) { + testDir := filepath.Join(mountPoint, "stress_directory") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(testDir) + + const numDirs = 500 + const numWorkers = 10 + + results := make(chan error, numWorkers) + + // Create directories concurrently + for w := 0; w < numWorkers; w++ { + go func(worker int) { + for i := worker; i < numDirs; i += numWorkers { + dirName := filepath.Join(testDir, fmt.Sprintf("dir_%d_%d", worker, i)) + err := os.MkdirAll(dirName, 0755) + if err != nil { + results <- err + return + } + + // Create a file in each directory + fileName := filepath.Join(dirName, "test.txt") + err = os.WriteFile(fileName, []byte("test"), 0644) + if err != nil { + results <- err + return + } + } + results <- nil + }(w) + } + + // Wait for all workers + for w := 0; w < numWorkers; w++ { + err := <-results + require.NoError(t, err) + } + + // Verify directory structure + dirs, err := filepath.Glob(filepath.Join(testDir, "dir_*")) + require.NoError(t, err) + require.Equal(t, numDirs, len(dirs)) +} + +// testEdgeCases tests various POSIX edge cases +func (s *ExternalPOSIXTestSuite) testEdgeCases(t *testing.T, mountPoint string) { + testDir := filepath.Join(mountPoint, "edge_cases") + err := os.MkdirAll(testDir, 0755) + require.NoError(t, err) + defer os.RemoveAll(testDir) + + t.Run("WriteToDirectoryAsFile", func(t *testing.T) { + // Test writing to a directory as if it were a file (should fail) + // Note: filepath.Join(testDir, "") returns testDir itself + err := os.WriteFile(testDir, []byte("test"), 0644) + require.Error(t, err) + // Verify the error is specifically about the target being a directory + var pathErr *os.PathError + require.ErrorAs(t, err, &pathErr) + require.Equal(t, syscall.EISDIR, pathErr.Err) + }) + + t.Run("VeryLongFileName", func(t *testing.T) { + // Test very long file names + longName := strings.Repeat("a", 255) // NAME_MAX is typically 255 + longFile := filepath.Join(testDir, longName) + + err := os.WriteFile(longFile, []byte("long name test"), 0644) + // This might succeed or fail depending on filesystem limits + if err == nil { + defer os.Remove(longFile) + t.Logf("Very long filename accepted") + } else { + t.Logf("Very long filename rejected: %v", err) + } + }) + + t.Run("SpecialCharacters", func(t *testing.T) { + // Test files with special characters + specialFiles := []string{ + "file with spaces.txt", + "file-with-dashes.txt", + "file_with_underscores.txt", + "file.with.dots.txt", + } + + for _, fileName := range specialFiles { + filePath := filepath.Join(testDir, fileName) + err := os.WriteFile(filePath, []byte("special char test"), 0644) + require.NoError(t, err, "Failed to create file with special chars: %s", fileName) + + // Verify we can read it back + _, err = os.ReadFile(filePath) + require.NoError(t, err, "Failed to read file with special chars: %s", fileName) + } + }) + + t.Run("DeepDirectoryNesting", func(t *testing.T) { + // Test deep directory nesting + deepPath := testDir + for i := 0; i < 100; i++ { + deepPath = filepath.Join(deepPath, fmt.Sprintf("level_%d", i)) + } + + err := os.MkdirAll(deepPath, 0755) + // This might fail due to PATH_MAX limits + if err != nil { + t.Logf("Deep directory nesting failed at some level: %v", err) + } else { + t.Logf("Deep directory nesting succeeded") + + // Test creating a file in the deep path + testFile := filepath.Join(deepPath, "deep_file.txt") + err = os.WriteFile(testFile, []byte("deep test"), 0644) + if err == nil { + t.Logf("File creation in deep path succeeded") + } else { + t.Logf("File creation in deep path failed: %v", err) + } + } + }) +} diff --git a/test/fuse_integration/sendfile_darwin.go b/test/fuse_integration/sendfile_darwin.go new file mode 100644 index 0000000000..1042ba7a08 --- /dev/null +++ b/test/fuse_integration/sendfile_darwin.go @@ -0,0 +1,45 @@ +//go:build darwin + +package fuse + +import ( + "syscall" + "unsafe" +) + +// Sendfile support for macOS + +func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) { + // macOS sendfile has different signature: sendfile(in_fd, out_fd, offset, len, hdtr, flags) + var off int64 + if offset != nil { + off = *offset + } + + length := int64(count) + + _, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE, + uintptr(inFd), // input fd + uintptr(outFd), // output fd + uintptr(off), // offset + uintptr(unsafe.Pointer(&length)), // length (in/out parameter) + 0, // hdtr (headers/trailers) + 0) // flags + + if errno != 0 { + return 0, errno + } + + // Update offset if provided + if offset != nil { + *offset += length + } + + return int(length), nil +} + +func isSendfileSupported() bool { + // macOS sendfile(2) requires a socket destination; file-to-file copies + // fail with ENOTSOCK. + return false +} diff --git a/test/fuse_integration/sendfile_linux.go b/test/fuse_integration/sendfile_linux.go new file mode 100644 index 0000000000..b5a0c0d610 --- /dev/null +++ b/test/fuse_integration/sendfile_linux.go @@ -0,0 +1,33 @@ +//go:build linux + +package fuse + +import ( + "syscall" + "unsafe" +) + +// Sendfile support for Linux + +func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) { + var offsetPtr uintptr + if offset != nil { + offsetPtr = uintptr(unsafe.Pointer(offset)) + } + + n, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE, + uintptr(outFd), + uintptr(inFd), + offsetPtr, + uintptr(count), + 0, 0) + + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +func isSendfileSupported() bool { + return true +} diff --git a/test/fuse_integration/sendfile_unsupported.go b/test/fuse_integration/sendfile_unsupported.go new file mode 100644 index 0000000000..64a7818dd2 --- /dev/null +++ b/test/fuse_integration/sendfile_unsupported.go @@ -0,0 +1,19 @@ +//go:build !linux && !darwin + +package fuse + +import ( + "errors" +) + +// Sendfile support for unsupported platforms + +var ErrSendfileNotSupported = errors.New("sendfile not supported on this platform") + +func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) { + return 0, ErrSendfileNotSupported +} + +func isSendfileSupported() bool { + return false +} diff --git a/test/fuse_integration/vectored_io_unix.go b/test/fuse_integration/vectored_io_unix.go new file mode 100644 index 0000000000..59b2b2cc47 --- /dev/null +++ b/test/fuse_integration/vectored_io_unix.go @@ -0,0 +1,126 @@ +//go:build unix + +package fuse + +import ( + "syscall" + "unsafe" +) + +// Vectored I/O support for Unix-like systems + +// IOVec represents an I/O vector for readv/writev operations +type IOVec struct { + Base *byte + Len uint64 +} + +func readvFile(fd int, iovs []IOVec) (int, error) { + if len(iovs) == 0 { + return 0, nil + } + + n, _, errno := syscall.Syscall(syscall.SYS_READV, + uintptr(fd), + uintptr(unsafe.Pointer(&iovs[0])), + uintptr(len(iovs))) + + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +func writevFile(fd int, iovs []IOVec) (int, error) { + if len(iovs) == 0 { + return 0, nil + } + + n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, + uintptr(fd), + uintptr(unsafe.Pointer(&iovs[0])), + uintptr(len(iovs))) + + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) { + if len(iovs) == 0 { + return 0, nil + } + + // preadv/pwritev may not be available on all Unix systems + // Fall back to individual pread calls + totalRead := 0 + currentOffset := offset + + for _, iov := range iovs { + if iov.Len == 0 { + continue + } + + buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len] + n, err := syscall.Pread(fd, buf, currentOffset) + if err != nil { + return totalRead, err + } + totalRead += n + currentOffset += int64(n) + + if n < int(iov.Len) { + break // EOF or partial read + } + } + + return totalRead, nil +} + +func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) { + if len(iovs) == 0 { + return 0, nil + } + + // preadv/pwritev may not be available on all Unix systems + // Fall back to individual pwrite calls + totalWritten := 0 + currentOffset := offset + + for _, iov := range iovs { + if iov.Len == 0 { + continue + } + + buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len] + n, err := syscall.Pwrite(fd, buf, currentOffset) + if err != nil { + return totalWritten, err + } + totalWritten += n + currentOffset += int64(n) + + if n < int(iov.Len) { + break // Partial write + } + } + + return totalWritten, nil +} + +// Helper function to create IOVec from byte slices +func makeIOVecs(buffers [][]byte) []IOVec { + iovs := make([]IOVec, len(buffers)) + for i, buf := range buffers { + if len(buf) > 0 { + iovs[i].Base = &buf[0] + iovs[i].Len = uint64(len(buf)) + } + } + return iovs +} + +func isVectoredIOSupported() bool { + return true +} diff --git a/test/fuse_integration/vectored_io_unsupported.go b/test/fuse_integration/vectored_io_unsupported.go new file mode 100644 index 0000000000..963c6e2d4a --- /dev/null +++ b/test/fuse_integration/vectored_io_unsupported.go @@ -0,0 +1,41 @@ +//go:build !unix + +package fuse + +import ( + "errors" +) + +// Vectored I/O support for unsupported platforms + +var ErrVectoredIONotSupported = errors.New("vectored I/O not supported on this platform") + +// IOVec represents an I/O vector for readv/writev operations +type IOVec struct { + Base *byte + Len uint64 +} + +func readvFile(fd int, iovs []IOVec) (int, error) { + return 0, ErrVectoredIONotSupported +} + +func writevFile(fd int, iovs []IOVec) (int, error) { + return 0, ErrVectoredIONotSupported +} + +func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) { + return 0, ErrVectoredIONotSupported +} + +func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) { + return 0, ErrVectoredIONotSupported +} + +func makeIOVecs(buffers [][]byte) []IOVec { + return nil +} + +func isVectoredIOSupported() bool { + return false +} diff --git a/test/fuse_integration/xattr_darwin.go b/test/fuse_integration/xattr_darwin.go new file mode 100644 index 0000000000..6bd6f2a908 --- /dev/null +++ b/test/fuse_integration/xattr_darwin.go @@ -0,0 +1,125 @@ +//go:build darwin + +package fuse + +import ( + "syscall" + "unsafe" +) + +// Extended attributes support for macOS + +const ( + // macOS-specific flags + XATTR_NOFOLLOW = 0x0001 + XATTR_CREATE = 0x0002 + XATTR_REPLACE = 0x0004 +) + +func setXattr(path, name string, value []byte, flags int) error { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + + var valuePtr unsafe.Pointer + if len(value) > 0 { + valuePtr = unsafe.Pointer(&value[0]) + } + + _, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + uintptr(valuePtr), + uintptr(len(value)), + uintptr(0), // position (not used for regular files) + uintptr(flags)) + + if errno != 0 { + return errno + } + return nil +} + +func getXattr(path, name string, value []byte) (int, error) { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return 0, err + } + + var valuePtr unsafe.Pointer + if len(value) > 0 { + valuePtr = unsafe.Pointer(&value[0]) + } + + size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + uintptr(valuePtr), + uintptr(len(value)), + uintptr(0), // position + uintptr(0)) // options + + if errno != 0 { + return 0, errno + } + return int(size), nil +} + +func listXattr(path string, list []byte) (int, error) { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + + var listPtr unsafe.Pointer + if len(list) > 0 { + listPtr = unsafe.Pointer(&list[0]) + } + + size, _, errno := syscall.Syscall6(syscall.SYS_LISTXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(listPtr), + uintptr(len(list)), + uintptr(0), // options + 0, 0) + + if errno != 0 { + return 0, errno + } + return int(size), nil +} + +func removeXattr(path, name string) error { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + + _, _, errno := syscall.Syscall6(syscall.SYS_REMOVEXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + uintptr(0), // options + 0, 0, 0) + + if errno != 0 { + return errno + } + return nil +} + +func isXattrSupported() bool { + return true +} diff --git a/test/fuse_integration/xattr_linux.go b/test/fuse_integration/xattr_linux.go new file mode 100644 index 0000000000..82f40d1c7f --- /dev/null +++ b/test/fuse_integration/xattr_linux.go @@ -0,0 +1,115 @@ +//go:build linux + +package fuse + +import ( + "syscall" + "unsafe" +) + +// Extended attributes support for Linux + +func setXattr(path, name string, value []byte, flags int) error { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + + var valuePtr unsafe.Pointer + if len(value) > 0 { + valuePtr = unsafe.Pointer(&value[0]) + } + + _, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + uintptr(valuePtr), + uintptr(len(value)), + uintptr(flags), + 0) + + if errno != 0 { + return errno + } + return nil +} + +func getXattr(path, name string, value []byte) (int, error) { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return 0, err + } + + var valuePtr unsafe.Pointer + if len(value) > 0 { + valuePtr = unsafe.Pointer(&value[0]) + } + + size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + uintptr(valuePtr), + uintptr(len(value)), + 0, + 0) + + if errno != 0 { + return 0, errno + } + return int(size), nil +} + +func listXattr(path string, list []byte) (int, error) { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + + var listPtr unsafe.Pointer + if len(list) > 0 { + listPtr = unsafe.Pointer(&list[0]) + } + + size, _, errno := syscall.Syscall(syscall.SYS_LISTXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(listPtr), + uintptr(len(list))) + + if errno != 0 { + return 0, errno + } + return int(size), nil +} + +func removeXattr(path, name string) error { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + nameBytes, err := syscall.BytePtrFromString(name) + if err != nil { + return err + } + + _, _, errno := syscall.Syscall(syscall.SYS_REMOVEXATTR, + uintptr(unsafe.Pointer(pathBytes)), + uintptr(unsafe.Pointer(nameBytes)), + 0) + + if errno != 0 { + return errno + } + return nil +} + +func isXattrSupported() bool { + return true +} diff --git a/test/fuse_integration/xattr_unsupported.go b/test/fuse_integration/xattr_unsupported.go new file mode 100644 index 0000000000..945fc2ff69 --- /dev/null +++ b/test/fuse_integration/xattr_unsupported.go @@ -0,0 +1,31 @@ +//go:build !linux && !darwin + +package fuse + +import ( + "errors" +) + +// Extended attributes support for unsupported platforms + +var ErrXattrNotSupported = errors.New("extended attributes not supported on this platform") + +func setXattr(path, name string, value []byte, flags int) error { + return ErrXattrNotSupported +} + +func getXattr(path, name string, value []byte) (int, error) { + return 0, ErrXattrNotSupported +} + +func listXattr(path string, list []byte) (int, error) { + return 0, ErrXattrNotSupported +} + +func removeXattr(path, name string) error { + return ErrXattrNotSupported +} + +func isXattrSupported() bool { + return false +} diff --git a/test/kafka/simple-publisher/go.mod b/test/kafka/simple-publisher/go.mod index 09309f0f23..de391da489 100644 --- a/test/kafka/simple-publisher/go.mod +++ b/test/kafka/simple-publisher/go.mod @@ -1,10 +1,10 @@ module simple-publisher -go 1.21 +go 1.25 require github.com/segmentio/kafka-go v0.4.47 require ( - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.20.3 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect ) diff --git a/test/kafka/simple-publisher/go.sum b/test/kafka/simple-publisher/go.sum index c9f731f2bb..82af443f27 100644 --- a/test/kafka/simple-publisher/go.sum +++ b/test/kafka/simple-publisher/go.sum @@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/test/s3/versioning/s3_listing_regression_test.go b/test/s3/versioning/s3_listing_regression_test.go new file mode 100644 index 0000000000..169c8511bb --- /dev/null +++ b/test/s3/versioning/s3_listing_regression_test.go @@ -0,0 +1,112 @@ +package s3api + +import ( + "bytes" + "context" + "fmt" + "sort" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Check both wire APIs with small pages. The directory walker alone does +// not exercise prefix normalization, delimiter-marker adjustment, or tokens. +func TestListingPrefixAndDelimiterPagination(t *testing.T) { + client := getS3Client(t) + ctx := context.Background() + for _, versioning := range []string{"", "Enabled", "Suspended"} { + t.Run("versioning="+versioning, func(t *testing.T) { + bucket := getNewBucketName() + createBucket(t, client, bucket) + defer deleteBucket(t, client, bucket) + if versioning != "" { + enableVersioning(t, client, bucket) + if versioning == "Suspended" { + _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended}, + }) + require.NoError(t, err) + } + } + for _, key := range []string{"a", "a-c", "a.txt", "a/b", "foo_bar", "foo_bar_baz", "foo_baz"} { + _, err := client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader([]byte("x"))}) + require.NoError(t, err) + } + for _, tc := range []struct { + name, prefix, delimiter, startAfter string + keys, prefixes []string + }{ + {"trailing slash", "a/", "", "", []string{"a/b"}, nil}, + {"trailing slash custom delimiter", "a/", "_", "", []string{"a/b"}, nil}, + {"custom delimiter", "foo", "_", "", nil, []string{"foo_"}}, + {"custom delimiter nested group", "foo_", "_", "", []string{"foo_bar", "foo_baz"}, []string{"foo_bar_"}}, + {"marker names directory", "a", "", "a", []string{"a-c", "a.txt", "a/b"}, nil}, + {"marker between name and slash", "a", "", "a.txt", []string{"a/b"}, nil}, + } { + for _, v2 := range []bool{false, true} { + for _, pageSize := range []int32{1, 2, 1000} { + t.Run(fmt.Sprintf("%s/v2=%t/max=%d", tc.name, v2, pageSize), func(t *testing.T) { + var keys, prefixes []string + marker, token := tc.startAfter, "" + for page := 0; ; page++ { + require.Less(t, page, 10, "listing must terminate without repeating a prefix") + var objects []types.Object + var commonPrefixes []types.CommonPrefix + var truncated bool + if v2 { + in := &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(tc.prefix), Delimiter: aws.String(tc.delimiter), MaxKeys: aws.Int32(pageSize)} + if token != "" { + in.ContinuationToken = aws.String(token) + } else if marker != "" { + in.StartAfter = aws.String(marker) + } + out, err := client.ListObjectsV2(ctx, in) + require.NoError(t, err) + objects, commonPrefixes, truncated = out.Contents, out.CommonPrefixes, aws.ToBool(out.IsTruncated) + next := aws.ToString(out.NextContinuationToken) + if truncated { + require.NotEmpty(t, next) + require.NotEqual(t, token, next, "token must advance") + } + token = next + } else { + out, err := client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: aws.String(bucket), Prefix: aws.String(tc.prefix), Delimiter: aws.String(tc.delimiter), MaxKeys: aws.Int32(pageSize), Marker: aws.String(marker), + }) + require.NoError(t, err) + objects, commonPrefixes, truncated = out.Contents, out.CommonPrefixes, aws.ToBool(out.IsTruncated) + next := aws.ToString(out.NextMarker) + if truncated { + if next == "" && len(objects) > 0 { + next = aws.ToString(objects[len(objects)-1].Key) + } + require.Greater(t, next, marker, "marker must advance") + } + marker = next + } + for _, object := range objects { + keys = append(keys, aws.ToString(object.Key)) + } + for _, prefix := range commonPrefixes { + prefixes = append(prefixes, aws.ToString(prefix.Prefix)) + } + if !truncated { + break + } + } + sort.Strings(keys) + sort.Strings(prefixes) + assert.Equal(t, tc.keys, keys) + assert.Equal(t, tc.prefixes, prefixes) + }) + } + } + } + }) + } +} diff --git a/weed/admin/handlers/file_browser_grpc.go b/weed/admin/handlers/file_browser_grpc.go index 0b2f9eb98d..9239688ed0 100644 --- a/weed/admin/handlers/file_browser_grpc.go +++ b/weed/admin/handlers/file_browser_grpc.go @@ -153,6 +153,9 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin ChunkSize: adminUploadChunkSize, MimeType: mimeType, AssignFunc: assignFunc, + LookupJwtFunc: func(lookupCtx context.Context, fileId string) security.EncodedJwt { + return operation.LookupJwt(lookupCtx, h.adminServer.GetMasterClient().GetMaster(lookupCtx), h.adminServer.GetGrpcDialOption(), fileId) + }, }) if err != nil { // Partial chunks come back even on error so we can clean them up rather diff --git a/weed/admin/plugin/plugin_scheduler.go b/weed/admin/plugin/plugin_scheduler.go index aa97e92737..58ad878d6a 100644 --- a/weed/admin/plugin/plugin_scheduler.go +++ b/weed/admin/plugin/plugin_scheduler.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -614,7 +615,7 @@ func deriveSchedulerAdminRuntime( descriptor *plugin_pb.JobTypeDescriptor, ) *plugin_pb.AdminRuntimeConfig { if cfg != nil && cfg.AdminRuntime != nil { - adminConfig := *cfg.AdminRuntime + adminConfig := proto.Clone(cfg.AdminRuntime).(*plugin_pb.AdminRuntimeConfig) // Overlay descriptor defaults for any zero numeric fields. Persisted // configs from older versions have no execution_timeout_seconds, and // without this overlay the scheduler would fall back to the 90s @@ -646,7 +647,7 @@ func deriveSchedulerAdminRuntime( adminConfig.ExecutionTimeoutSeconds = defaults.ExecutionTimeoutSeconds } } - return &adminConfig + return adminConfig } if descriptor == nil || descriptor.AdminRuntimeDefaults == nil { diff --git a/weed/admin/plugin/registry.go b/weed/admin/plugin/registry.go index 2201e6a9c5..6e27d30411 100644 --- a/weed/admin/plugin/registry.go +++ b/weed/admin/plugin/registry.go @@ -8,6 +8,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/proto" ) const defaultWorkerStaleTimeout = 2 * time.Minute @@ -495,36 +496,35 @@ func cloneJobTypeCapability(in *plugin_pb.JobTypeCapability) *plugin_pb.JobTypeC if in == nil { return nil } - out := *in - return &out + return proto.Clone(in).(*plugin_pb.JobTypeCapability) } func cloneWorkerHeartbeat(in *plugin_pb.WorkerHeartbeat) *plugin_pb.WorkerHeartbeat { if in == nil { return nil } - out := *in + out := proto.Clone(in).(*plugin_pb.WorkerHeartbeat) if in.RunningWork != nil { - out.RunningWork = make([]*plugin_pb.RunningWork, 0, len(in.RunningWork)) - for _, rw := range in.RunningWork { - if rw == nil { - continue + running := out.RunningWork[:0] + // Clone can turn nil messages into empty messages, so filter using + // their original positions while reusing the cloned entries. + for i, work := range in.RunningWork { + if work != nil { + running = append(running, out.RunningWork[i]) } - clone := *rw - out.RunningWork = append(out.RunningWork, &clone) } - } - if in.QueuedJobsByType != nil { - out.QueuedJobsByType = make(map[string]int32, len(in.QueuedJobsByType)) - for k, v := range in.QueuedJobsByType { - out.QueuedJobsByType[k] = v + clear(out.RunningWork[len(running):]) + if running == nil { + running = []*plugin_pb.RunningWork{} } + out.RunningWork = running } - if in.Metadata != nil { - out.Metadata = make(map[string]string, len(in.Metadata)) - for k, v := range in.Metadata { - out.Metadata[k] = v - } + // Preserve writable empty maps without rebuilding populated clones. + if in.QueuedJobsByType != nil && out.QueuedJobsByType == nil { + out.QueuedJobsByType = make(map[string]int32) } - return &out + if in.Metadata != nil && out.Metadata == nil { + out.Metadata = make(map[string]string) + } + return out } diff --git a/weed/admin/plugin/registry_snapshot_test.go b/weed/admin/plugin/registry_snapshot_test.go new file mode 100644 index 0000000000..1546096048 --- /dev/null +++ b/weed/admin/plugin/registry_snapshot_test.go @@ -0,0 +1,65 @@ +package plugin + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestCloneWorkerHeartbeatFiltersNilAndIsolatesState(t *testing.T) { + t.Parallel() + source := &plugin_pb.WorkerHeartbeat{ + RunningWork: []*plugin_pb.RunningWork{nil, {WorkId: "first"}, nil, {WorkId: "second"}, nil}, + QueuedJobsByType: map[string]int32{"vacuum": 2}, + Metadata: map[string]string{"rack": "rack-a"}, + } + got := cloneWorkerHeartbeat(source) + if len(got.RunningWork) != 2 || got.RunningWork[0].GetWorkId() != "first" || got.RunningWork[1].GetWorkId() != "second" { + t.Fatalf("unexpected filtered work: %v", got.RunningWork) + } + got.RunningWork[0].WorkId = "changed" + got.QueuedJobsByType["vacuum"] = 99 + got.Metadata["rack"] = "changed" + if len(source.RunningWork) != 5 || source.RunningWork[0] != nil || source.RunningWork[1].WorkId != "first" { + t.Fatalf("cloning mutated source work: %v", source.RunningWork) + } + if source.QueuedJobsByType["vacuum"] != 2 || source.Metadata["rack"] != "rack-a" { + t.Fatal("cloned maps alias the source") + } +} + +func TestCloneWorkerHeartbeatPreservesEmptyCollections(t *testing.T) { + t.Parallel() + if cloneWorkerHeartbeat(nil) != nil { + t.Fatal("nil heartbeat must stay nil") + } + for _, tc := range []struct { + name string + source *plugin_pb.WorkerHeartbeat + wantNilWork bool + }{ + {"nil", &plugin_pb.WorkerHeartbeat{}, true}, + {"empty", &plugin_pb.WorkerHeartbeat{ + RunningWork: []*plugin_pb.RunningWork{}, + QueuedJobsByType: map[string]int32{}, + Metadata: map[string]string{}, + }, false}, + {"all nil work", &plugin_pb.WorkerHeartbeat{RunningWork: []*plugin_pb.RunningWork{nil, nil}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got := cloneWorkerHeartbeat(tc.source) + if len(got.RunningWork) != 0 || (got.RunningWork == nil) != tc.wantNilWork { + t.Fatalf("unexpected empty work slice: %#v", got.RunningWork) + } + if (got.QueuedJobsByType == nil) != (tc.source.QueuedJobsByType == nil) || (got.Metadata == nil) != (tc.source.Metadata == nil) { + t.Fatal("clone changed whether empty maps are writable") + } + if got.Metadata != nil { + got.Metadata["added"] = "value" + if len(tc.source.Metadata) != 0 { + t.Fatal("empty metadata map aliases source") + } + } + }) + } +} diff --git a/weed/cluster/lock_ring_manager.go b/weed/cluster/lock_ring_manager.go index 036db3c4f9..04577ac18b 100644 --- a/weed/cluster/lock_ring_manager.go +++ b/weed/cluster/lock_ring_manager.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/protobuf/proto" ) const LockRingStabilizationInterval = 1 * time.Second @@ -102,9 +103,7 @@ func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb. if !ok || update == nil { return nil } - cp := *update - cp.Servers = append([]string(nil), update.Servers...) - return &cp + return proto.Clone(update).(*master_pb.LockRingUpdate) } // scheduleBroadcast resets the stabilization timer. If another change arrives diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index ad0dbadc4c..0ee3f03136 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -238,7 +238,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { } var jwtAuthorization security.EncodedJwt if isSecure { - jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid) + jwtAuthorization = operation.LookupJwt(context.Background(), b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid) } if e := util_http.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil { s.completed++ diff --git a/weed/command/filer.go b/weed/command/filer.go index 90eb04bee9..245182117a 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -51,6 +51,15 @@ var ( // S3 gateway, whose remote-mount read paths dial the mounted endpoint directly. const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only entry accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints." +// iamSelfServiceRequiresGrantUsage documents the flag shared by every command +// that runs the embedded IAM API. +const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity." + +// allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal +// object-key character instead of normalizing it to '/'. Repeated '//' in a +// key still collapses: a filer entry name cannot contain an empty segment. +const allowBackslashInKeyUsage = "treat backslash as a regular object-key character instead of converting it to '/' (aws-compatible; '//' still collapses)" + type FilerOptions struct { masters *pb.ServerDiscovery mastersString *string @@ -173,6 +182,8 @@ func init() { filerS3Options.cacheSizeMB = cmdFiler.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)") filerS3Options.readerCacheSizeMB = cmdFiler.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) + filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) + filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) // start webdav on filer filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") @@ -191,6 +202,7 @@ func init() { filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service") filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address") filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port") + filerIamOptions.selfServiceRequiresGrant = filerS3Options.iamSelfServiceRequiresGrant filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server") filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port") diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 7e22549316..569ef62430 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -214,11 +214,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour } if filer_pb.IsCreate(resp) { if message.NewParentPath == option.bucketsDir { + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } return handleCreateBucket(message.NewEntry) } if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } // Propagate delete markers as deletions on the remote. // Delete markers are zero-content version entries, so they // would be filtered out by the HasData check below. @@ -292,6 +298,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour return nil } if resp.Directory == option.bucketsDir { + if isSystemPath(resp.Directory, message.OldEntry.Name) { + return nil + } return handleDeleteBucket(message.OldEntry) } // Skip deletion of internal version files; individual version @@ -300,6 +309,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name) return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) { + return nil + } bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory) if !ok { return nil @@ -323,6 +335,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour if message.OldEntry.Name == message.NewEntry.Name { return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) || isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } if err := handleCreateBucket(message.NewEntry); err != nil { return err } @@ -339,6 +354,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name) return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory) newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath) if oldOk && newOk { diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index da6650f462..d36147191f 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -144,6 +144,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } // Propagate delete markers as deletions on the remote. // Delete markers are zero-content version entries, so they // would be filtered out by the HasData check below. @@ -229,6 +233,10 @@ func processDeleteEvent( glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name) return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) { + glog.V(2).Infof("skipping delete of system path: %s/%s", resp.Directory, message.OldEntry.Name) + return nil + } if filer.IsMetadataOnlyDelete(message.OldEntry) { glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name) return nil @@ -339,6 +347,10 @@ func processUpdateEvent( glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name) return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping update of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()} @@ -428,7 +440,7 @@ func processUpdateEvent( } glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest)) if err := client.DeleteFile(oldDest); err != nil { - if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) { + if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) || isSystemPath(resp.Directory, message.OldEntry.Name) { return nil } if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { @@ -881,6 +893,23 @@ func isMultipartUploadDir(dir string) bool { strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/") } +// isSystemPath returns true if the dir and name refer to the internal .system folder +// under /buckets (such as the bucket-owner index at /buckets/.system/owners/...) +// or any entry inside it. These paths are internal to SeaweedFS and must not +// be synced to remote storage. +func isSystemPath(dir string, name string) bool { + fullPath := string(util.NewFullPath(dir, name)) + trimmed := strings.Trim(fullPath, "/") + parts := strings.Split(trimmed, "/") + if len(parts) >= 1 && parts[0] == s3_constants.SystemFolder { + return true + } + if len(parts) >= 2 && parts[1] == s3_constants.SystemFolder { + return true + } + return false +} + // isDeleteMarker returns true if the entry is an S3 delete marker // (a zero-content version entry with ExtDeleteMarkerKey set to "true"). func isDeleteMarker(entry *filer_pb.Entry) bool { diff --git a/weed/command/filer_remote_sync_dir_test.go b/weed/command/filer_remote_sync_dir_test.go index d7fb99ea3f..93c8340b47 100644 --- a/weed/command/filer_remote_sync_dir_test.go +++ b/weed/command/filer_remote_sync_dir_test.go @@ -2016,3 +2016,74 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) { t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry) } } + +func TestIsSystemPath(t *testing.T) { + tests := []struct { + dir string + name string + expected bool + }{ + {dir: "/buckets", name: ".system", expected: true}, + {dir: "/buckets/.system", name: "owners", expected: true}, + {dir: "/buckets/.system/owners", name: "alice", expected: true}, + {dir: "/buckets/.system/owners/alice", name: "mybucket", expected: true}, + {dir: "/", name: ".system", expected: true}, + {dir: "/.system", name: "config", expected: true}, + {dir: "/custom-mount", name: ".system", expected: true}, + {dir: "/custom-mount/.system/owners", name: "alice", expected: true}, + // Bucket contents should not match + {dir: "/buckets/mybucket", name: "file.txt", expected: false}, + {dir: "/buckets/mybucket", name: ".system", expected: false}, + {dir: "/buckets/mybucket/.system", name: "data", expected: false}, + {dir: "/buckets", name: ".system-other", expected: false}, + {dir: "/buckets/.system-other", name: "sub", expected: false}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/%s", tt.dir, tt.name), func(t *testing.T) { + got := isSystemPath(tt.dir, tt.name) + if got != tt.expected { + t.Errorf("isSystemPath(%q, %q) = %v, want %v", tt.dir, tt.name, got, tt.expected) + } + }) + } +} + +func TestRemoteSyncSkipsSystemFolder(t *testing.T) { + const mountedDir = "/buckets" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "azure", Bucket: "project-storage", Path: "/"} + + // Test update event on /buckets/.system/owners/... is skipped + updateResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/buckets/.system/owners/structify-api", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + NewParentPath: "/buckets/.system/owners/structify-api", + NewEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + }, + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{} + if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, updateResp); err != nil { + t.Fatalf("processUpdateEvent failed: %v", err) + } + if len(remote.writes) != 0 || len(remote.metadataUpdates) != 0 { + t.Fatalf("expected 0 remote writes or metadata updates for system path, got %d writes and %d metadata updates", + len(remote.writes), len(remote.metadataUpdates)) + } + + // Test delete event on /buckets/.system/owners/... is skipped + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/buckets/.system/owners/structify-api", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + }, + } + if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleteResp); err != nil { + t.Fatalf("processDeleteEvent failed: %v", err) + } + if len(remote.deletes) != 0 { + t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes)) + } +} + diff --git a/weed/command/iam.go b/weed/command/iam.go index 77f3a9014a..ce11f56ab2 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -28,10 +28,11 @@ var ( ) type IamOptions struct { - filer *string - masters *string - ip *string - port *int + filer *string + masters *string + ip *string + port *int + selfServiceRequiresGrant *bool } func init() { @@ -40,6 +41,7 @@ func init() { iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers") iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address") iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port") + iamStandaloneOptions.selfServiceRequiresGrant = cmdIam.Flag.Bool("selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) } var cmdIam = &Command{ @@ -101,10 +103,11 @@ func (iamopt *IamOptions) startIamServer() bool { masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap() router := mux.NewRouter().SkipClean(true) iamApiServer, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{ - Masters: masters, - Filers: filerAddresses, - Port: *iamopt.port, - GrpcDialOption: grpcDialOption, + Masters: masters, + Filers: filerAddresses, + Port: *iamopt.port, + GrpcDialOption: grpcDialOption, + IamSelfServiceRequiresGrant: *iamopt.selfServiceRequiresGrant, }) glog.V(0).Info("NewIamApiServer created") if iamApiServer_err != nil { diff --git a/weed/command/mini.go b/weed/command/mini.go index 1ded2f936c..9fd8d0963f 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -556,6 +556,8 @@ func initMiniS3Flags() { miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)") miniS3Options.readerCacheSizeMB = cmdMini.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) + miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) + miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) // In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server miniS3Options.debug = new(bool) // explicitly false miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)") diff --git a/weed/command/s3.go b/weed/command/s3.go index 036e3813eb..958afb8eab 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -84,6 +84,8 @@ type S3Options struct { readerCacheSizeMB *int64 allowUntrustedRemoteEndpoints *bool + allowBackslashInKey *bool + iamSelfServiceRequiresGrant *bool // shutdownCtx, when non-nil, tells startS3Server to gracefully shut down // its HTTP/gRPC servers once the ctx is cancelled, in addition to on // interrupt. Used by weed mini to orchestrate an ordered shutdown. @@ -140,6 +142,8 @@ func init() { s3StandaloneOptions.cacheSizeMB = cmdS3.Flag.Int64("cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)") s3StandaloneOptions.readerCacheSizeMB = cmdS3.Flag.Int64("readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) + s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage) + s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) } var cmdS3 = &Command{ @@ -337,7 +341,8 @@ func (s3opt *S3Options) startS3Server() bool { router := mux.NewRouter().SkipClean(true) var localFilerSocket string - if s3opt.localFilerSocket != nil { + // The filer only listens on its local socket outside Windows. + if s3opt.localFilerSocket != nil && runtime.GOOS != "windows" { localFilerSocket = *s3opt.localFilerSocket } var s3ApiServer *s3api.S3ApiServer @@ -399,6 +404,8 @@ func (s3opt *S3Options) startS3Server() bool { MaxMB: filerMaxMB, AllowUntrustedRemoteEndpoints: *s3opt.allowUntrustedRemoteEndpoints, + AllowBackslashInKey: *s3opt.allowBackslashInKey, + IamSelfServiceRequiresGrant: *s3opt.iamSelfServiceRequiresGrant, }) if s3ApiServer_err != nil { glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 67bfea93aa..8b9552e052 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -33,12 +33,22 @@ recursive_delete = false # faster than previous leveldb, recommended. enabled = true dir = "./filerldb2" # directory to store level db files +# optional tuning, applied to each of the 8 partitioned stores +# block_cache_size_mb = 32 # sorted-table read cache +# write_buffer_size_mb = 16 # memtable size before flushing to disk +# block_size_kb = 4 # uncompressed block ('page') size inside table files +# compaction_table_size_mb = 2 # target size of generated table files [leveldb3] # similar to leveldb2. # each bucket has its own meta store. enabled = false dir = "./filerldb3" # directory to store level db files +# optional tuning, applied to the default store and each bucket store +# block_cache_size_mb = 32 # sorted-table read cache +# write_buffer_size_mb = 16 # memtable size before flushing to disk +# block_size_kb = 4 # uncompressed block ('page') size inside table files +# compaction_table_size_mb = 2 # target size of generated table files [rocksdb] # local on disk, similar to leveldb diff --git a/weed/command/server.go b/weed/command/server.go index 0e4ac169a3..a9b27bc064 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -195,10 +195,12 @@ func init() { s3Options.iamReadOnly = cmdServer.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server") s3Options.cipher = cmdServer.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads") s3Options.externalUrl = cmdServer.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Advertised to Iceberg and Lance clients, and tried first when verifying S3 signatures behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.") + s3Options.allowBackslashInKey = cmdServer.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) s3Options.defaultFileMode = cmdServer.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666") s3Options.cacheSizeMB = cmdServer.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)") s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) + s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port") sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication") @@ -214,6 +216,7 @@ func init() { sftpOptions.dataCenter = cmdServer.Flag.String("sftp.dataCenter", "", "prefer to read and write to volumes in this data center") sftpOptions.localSocket = cmdServer.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-.sock") iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port") + iamOptions.selfServiceRequiresGrant = s3Options.iamSelfServiceRequiresGrant webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") diff --git a/weed/command/shell.go b/weed/command/shell.go index 0162d15015..f0aa1a30a7 100644 --- a/weed/command/shell.go +++ b/weed/command/shell.go @@ -3,6 +3,7 @@ package command import ( "fmt" "os" + "time" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -12,10 +13,11 @@ import ( ) var ( - shellOptions shell.ShellOptions - shellInitialFiler *string - shellCluster *string - shellDebug *bool + shellOptions shell.ShellOptions + shellInitialFiler *string + shellCluster *string + shellDebug *bool + shellConnectTimeout *time.Duration ) func init() { @@ -25,6 +27,7 @@ func init() { shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888") shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml") shellDebug = cmdShell.Flag.Bool("debug", false, "print informational logs to stderr") + shellConnectTimeout = cmdShell.Flag.Duration("connectTimeout", 30*time.Second, "timeout for the initial master connection and filer discovery") } var cmdShell = &Command{ @@ -64,6 +67,7 @@ func runShell(command *Command, args []string) bool { } shellOptions.FilerAddress = pb.ServerAddress(filerAddress) shellOptions.Debug = *shellDebug + shellOptions.ConnectTimeout = *shellConnectTimeout if shellOptions.Debug { fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress) } diff --git a/weed/command/upload.go b/weed/command/upload.go index bb542a030c..def64adc77 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" @@ -31,6 +32,7 @@ type UploadOptions struct { diskType *string maxMB *int usePublicUrl *bool + concurrency *int } func init() { @@ -46,6 +48,7 @@ func init() { upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit") upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server") + upload.concurrency = cmdUpload.Flag.Int("c", 8, "concurrent file upload goroutines when uploading a folder") } var cmdUpload = &Command{ @@ -85,6 +88,7 @@ func runUpload(cmd *Command, args []string) bool { if *upload.dir == "" { return false } + var filePaths []string err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error { if err == nil { if !info.IsDir() { @@ -93,23 +97,7 @@ func runUpload(cmd *Command, args []string) bool { return nil } } - parts, e := operation.NewFileParts([]string{path}) - if e != nil { - return e - } - results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{ - Replication: *upload.replication, - Collection: *upload.collection, - DataCenter: *upload.dataCenter, - Ttl: *upload.ttl, - DiskType: *upload.diskType, - MaxMB: *upload.maxMB, - }, *upload.usePublicUrl) - bytes, _ := json.Marshal(results) - fmt.Println(string(bytes)) - if e != nil { - return e - } + filePaths = append(filePaths, path) } } else { fmt.Println(err) @@ -118,6 +106,72 @@ func runUpload(cmd *Command, args []string) bool { }) if err != nil { fmt.Println(err.Error()) + } + + concurrency := *upload.concurrency + if concurrency < 1 { + concurrency = 1 + } + masterFn := func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) } + pref := operation.StoragePreference{ + Replication: *upload.replication, + Collection: *upload.collection, + DataCenter: *upload.dataCenter, + Ttl: *upload.ttl, + DiskType: *upload.diskType, + MaxMB: *upload.maxMB, + } + results := make([][]operation.SubmitResult, len(filePaths)) + ready := make([]chan struct{}, len(filePaths)) + for index := range ready { + ready[index] = make(chan struct{}) + } + printDone := make(chan bool, 1) + go func() { + failed := false + for index := range ready { + <-ready[index] + bytes, _ := json.Marshal(results[index]) + fmt.Println(string(bytes)) + for _, r := range results[index] { + if r.Error != "" { + failed = true + } + } + } + printDone <- failed + }() + + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + for index, filePath := range filePaths { + sem <- struct{}{} + wg.Add(1) + go func(index int, filePath string) { + defer close(ready[index]) + defer wg.Done() + defer func() { <-sem }() + parts, e := operation.NewFileParts([]string{filePath}) + if e != nil { + results[index] = []operation.SubmitResult{{FileName: filepath.Base(filePath), Error: e.Error()}} + return + } + fileResults, e := operation.SubmitFiles(masterFn, grpcDialOption, parts, pref, *upload.usePublicUrl) + if e != nil { + for i := range fileResults { + if fileResults[i].Error == "" { + fileResults[i].Error = e.Error() + } + } + } + results[index] = fileResults + }(index, filePath) + } + wg.Wait() + if <-printDone { + return false + } + if err != nil { return false } } else { @@ -134,12 +188,12 @@ func runUpload(cmd *Command, args []string) bool { DiskType: *upload.diskType, MaxMB: *upload.maxMB, }, *upload.usePublicUrl) + bytes, _ := json.Marshal(results) + fmt.Println(string(bytes)) if err != nil { fmt.Println(err.Error()) return false } - bytes, _ := json.Marshal(results) - fmt.Println(string(bytes)) } return true } diff --git a/weed/credential/credential_manager.go b/weed/credential/credential_manager.go index ec37b2a8e0..7bf84c9670 100644 --- a/weed/credential/credential_manager.go +++ b/weed/credential/credential_manager.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/wdclient" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" ) // FilerAddressSetter is an interface for credential stores that need a dynamic filer address @@ -168,15 +169,16 @@ func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_ staticNames := cm.staticNames cm.staticMu.RUnlock() if len(staticNames) > 0 { - var dynamicOnly []*iam_pb.Identity - for _, ident := range config.Identities { + configCopy := proto.Clone(config).(*iam_pb.S3ApiConfiguration) + dynamicOnly := configCopy.Identities[:0] + for _, ident := range configCopy.Identities { if !staticNames[ident.Name] { dynamicOnly = append(dynamicOnly, ident) } } - configCopy := *config + clear(configCopy.Identities[len(dynamicOnly):]) configCopy.Identities = dynamicOnly - return cm.Store.SaveConfiguration(ctx, &configCopy) + return cm.Store.SaveConfiguration(ctx, configCopy) } return cm.Store.SaveConfiguration(ctx, config) } diff --git a/weed/credential/credential_manager_snapshot_test.go b/weed/credential/credential_manager_snapshot_test.go new file mode 100644 index 0000000000..41c6d4628d --- /dev/null +++ b/weed/credential/credential_manager_snapshot_test.go @@ -0,0 +1,46 @@ +package credential + +import ( + "context" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "google.golang.org/protobuf/proto" +) + +type snapshotConfigurationStore struct { + CredentialStore + saved *iam_pb.S3ApiConfiguration +} + +func (s *snapshotConfigurationStore) SaveConfiguration(_ context.Context, config *iam_pb.S3ApiConfiguration) error { + s.saved = config + return nil +} + +func TestSaveConfigurationFiltersStaticIdentitySnapshot(t *testing.T) { + t.Parallel() + store := &snapshotConfigurationStore{} + manager := &CredentialManager{Store: store} + manager.SetStaticIdentities([]*iam_pb.Identity{{Name: "static"}}) + source := &iam_pb.S3ApiConfiguration{Identities: []*iam_pb.Identity{ + {Name: "alice", Actions: []string{"Read"}}, + {Name: "static"}, + {Name: "bob", Actions: []string{"Write"}}, + }} + before := proto.Clone(source) + if err := manager.SaveConfiguration(context.Background(), source); err != nil { + t.Fatal(err) + } + if store.saved == nil || len(store.saved.Identities) != 2 || store.saved.Identities[0].Name != "alice" || store.saved.Identities[1].Name != "bob" { + t.Fatalf("unexpected saved configuration: %v", store.saved) + } + if !proto.Equal(source, before) { + t.Fatal("filtering mutated the caller's configuration") + } + store.saved.Identities[0].Actions[0] = "Admin" + store.saved.Identities[1].Name = "changed" + if !proto.Equal(source, before) { + t.Fatal("saved identities alias the caller's configuration") + } +} diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index e358727743..52f959d80a 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -23,14 +23,20 @@ type SqlGenerator interface { GetSqlDeleteFolderChildren(tableName string) string GetSqlListExclusive(tableName string) string GetSqlListInclusive(tableName string) string + // GetSqlScanPage returns the query and arguments selecting dirhash, name, + // directory and meta of up to limit rows after the (dirhash, name) cursor. + GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) GetSqlCreateTable(tableName string) string GetSqlDropTable(tableName string) string } type AbstractSqlStore struct { SqlGenerator - DB *sql.DB - KvDB *sql.DB + DB *sql.DB + KvDB *sql.DB + // ScanDB, when set, is the pool snapshot scans run on (SQLite's main pool + // is a single connection that a long scan must not hold). + ScanDB *sql.DB SupportBucketTable bool SkipDDL bool dbs map[string]bool @@ -457,6 +463,9 @@ func (store *AbstractSqlStore) Shutdown() { if store.KvDB != nil { store.KvDB.Close() } + if store.ScanDB != nil { + store.ScanDB.Close() + } } func isValidBucket(bucket string) bool { diff --git a/weed/filer/abstract_sql/abstract_sql_store_scan.go b/weed/filer/abstract_sql/abstract_sql_store_scan.go new file mode 100644 index 0000000000..2028643755 --- /dev/null +++ b/weed/filer/abstract_sql/abstract_sql_store_scan.go @@ -0,0 +1,368 @@ +package abstract_sql + +import ( + "context" + "database/sql" + "fmt" + "math" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +var _ filer.EntryScanner = (*AbstractSqlStore)(nil) + +// scanTable is one table to read during a scan together with the path its +// directory column is relative to: "" for the default table, whose rows hold +// full directories, and "/buckets/" for a per-bucket table. +type scanTable struct { + table string + base string + // cachedOnly marks a table that only a stale bucket cache remembers: its + // bucket directory was already gone from the snapshot, so a missing table + // means it was dropped meanwhile, not that rows are absent. + cachedOnly bool +} + +// scanPageSize is how many rows one keyset page of a scan fetches. +var scanPageSize = 1000 + +// SetScanPageSizeForTest lets tests drive the keyset cursor across pages. It +// returns the previous size. +func SetScanPageSizeForTest(n int) int { + prev := scanPageSize + scanPageSize = n + return prev +} + +// ScanEntries implements filer.EntryScanner: every table that can hold an +// entry below dirPath is read inside one REPEATABLE READ, read-only +// transaction, in primary-key pages. All rows are seen as of one instant; a +// rename committed meanwhile shows up under exactly one of its names. The +// transaction takes no row locks and does not block writers; it does hold the +// tables it has read against DDL, so a bucket deleted while the scan runs +// drops its table once the scan ends. A page's cursor is closed before its +// entries go to fn, so fn may read the store through the context it receives +// (which carries the transaction) on the same connection. +func (store *AbstractSqlStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error { + // Read before the transaction holds a connection: OnBucketCreation keeps + // dbsLock while it waits for a connection of its own, so taking the lock + // while holding a connection could deadlock on an exhausted pool. + cachedBuckets := store.cachedBucketNames() + + tx, err := store.beginSnapshotTx(ctx) + if err != nil { + return err + } + // Read-only: nothing to commit, and a rollback releases the snapshot. + defer tx.Rollback() + + tables, err := store.scanTables(ctx, tx, dirPath, cachedBuckets) + if err != nil { + return err + } + // Reads made through this context (getTxOrDB, kvDB) use the transaction + // and therefore the snapshot. + txCtx := context.WithValue(ctx, "tx", tx) + // The root's own row may live in a table the scan does not read (a + // bucket directory is in the default table, its contents in the bucket's). + if root, err := store.readRowInSnapshot(txCtx, tx, dirPath); err != nil { + return err + } else if root != nil { + if err := fn(txCtx, root); err != nil { + return err + } + } + for _, t := range tables { + if err := store.scanOneTable(txCtx, tx, t, dirPath, fn); err != nil { + return err + } + } + return nil +} + +// readRowInSnapshot reads one entry's row inside the transaction, routing to +// the table getTxOrDB would use but without its bucket-table creation. A +// missing row is nil. +func (store *AbstractSqlStore) readRowInSnapshot(ctx context.Context, tx *sql.Tx, p util.FullPath) (*filer.Entry, error) { + if p == "/" { + return nil, nil + } + parent, name := p.DirAndName() + table, shortDir := DEFAULT_TABLE, parent + if store.SupportBucketTable { + if bucket := bucketTableFor(util.FullPath(parent)); bucket != "" { + table = bucket + if shortDir = strings.TrimPrefix(parent, bucketBase(bucket)); shortDir == "" { + shortDir = "/" + } + } + } + var data []byte + err := tx.QueryRowContext(ctx, store.GetSqlFind(table), util.HashStringToLong(shortDir), name, shortDir).Scan(&data) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, missingTableOr(err, table, fmt.Sprintf("read %s", p)) + } + entry := &filer.Entry{FullPath: p} + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + return nil, fmt.Errorf("decode %s: %v", p, err) + } + return entry, nil +} + +// beginSnapshotTx starts the read-only REPEATABLE READ transaction that gives +// one snapshot across statements. Nothing weaker is tried: under READ +// COMMITTED each page would see a different snapshot, and the stream would +// still be advertised as one. +func (store *AbstractSqlStore) beginSnapshotTx(ctx context.Context) (*sql.Tx, error) { + db := store.DB + if store.ScanDB != nil { + db = store.ScanDB + } else if db.Stats().MaxOpenConnections == 1 { + // The scan would hold the filer's only connection for its whole run. + return nil, fmt.Errorf("%w: the store's single connection cannot be held for a scan", filer.ErrUnsupportedEntryScan) + } + // A BeginTx failure (a down database) must fail the traversal, not route it + // to the walk: a recovered database lets the walk read renamed-away entries + // as absent, the very loss this scan exists to prevent. + tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if err != nil { + return nil, fmt.Errorf("begin snapshot: %v", err) + } + return tx, nil +} + +func (store *AbstractSqlStore) cachedBucketNames() []string { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + names := make([]string, 0, len(store.dbs)) + for name := range store.dbs { + names = append(names, name) + } + return names +} + +// scanTables picks the tables to read for dirPath: the default table alone, +// one bucket's table for a path inside that bucket, or the default table plus +// every bucket table (enumerated from /buckets inside the snapshot). +func (store *AbstractSqlStore) scanTables(ctx context.Context, tx *sql.Tx, dirPath util.FullPath, cachedBuckets []string) ([]scanTable, error) { + if !store.SupportBucketTable { + return []scanTable{{table: DEFAULT_TABLE}}, nil + } + if bucket := bucketTableFor(dirPath); bucket != "" { + return []scanTable{{table: bucket, base: bucketBase(bucket)}}, nil + } + tables := []scanTable{{table: DEFAULT_TABLE}} + if dirPath != "/" && string(dirPath) != bucketsRoot { + // Outside every bucket (e.g. /topics, or an internal /buckets/.system + // folder): everything below dirPath is in the default table. + return tables, nil + } + buckets, err := store.listBucketNames(ctx, tx, cachedBuckets) + if err != nil { + return nil, err + } + for _, bucket := range buckets.names { + tables = append(tables, scanTable{table: bucket, base: bucketBase(bucket), cachedOnly: !buckets.seen[bucket]}) + } + return tables, nil +} + +const bucketsRoot = "/buckets" + +func bucketBase(bucket string) string { + return bucketsRoot + "/" + bucket +} + +// bucketTableFor returns the bucket whose table alone holds everything below +// dirPath, or "" when dirPath is not inside a real bucket (as getTxOrDB routes). +func bucketTableFor(dirPath util.FullPath) string { + p := string(dirPath) + if !strings.HasPrefix(p, bucketsRoot+"/") { + return "" + } + rest := p[len(bucketsRoot)+1:] + bucket := rest + if i := strings.Index(rest, "/"); i >= 0 { + bucket = rest[:i] + } + if bucket == "" || strings.HasPrefix(bucket, ".") || !isValidBucket(bucket) { + return "" + } + return bucket +} + +// bucketNames is the snapshot-listed and cached-only buckets whose tables a +// scan may read. +type bucketNames struct { + names []string + seen map[string]bool // listed under /buckets inside the snapshot +} + +// listBucketNames reads the directories under /buckets within the snapshot +// and keeps those that route to their own table, plus the buckets this +// process has already seen (their directory entry may be gone, their rows not). +func (store *AbstractSqlStore) listBucketNames(ctx context.Context, tx *sql.Tx, cachedBuckets []string) (bucketNames, error) { + var names bucketNames + names.seen = make(map[string]bool) + added := make(map[string]bool) + add := func(name string) { + if added[name] || strings.HasPrefix(name, ".") || !isValidBucket(name) { + return + } + added[name] = true + names.names = append(names.names, name) + } + + const page = 1000 + startFrom := "" + for { + type row struct { + name string + data []byte + } + var rows []row + if err := func() error { + res, err := tx.QueryContext(ctx, store.GetSqlListExclusive(DEFAULT_TABLE), util.HashStringToLong(bucketsRoot), startFrom, bucketsRoot, "%", page) + if err != nil { + return fmt.Errorf("list %s: %v", bucketsRoot, err) + } + defer res.Close() + for res.Next() { + var r row + if err := res.Scan(&r.name, &r.data); err != nil { + return fmt.Errorf("scan %s: %v", bucketsRoot, err) + } + rows = append(rows, r) + } + return res.Err() + }(); err != nil { + return bucketNames{}, err + } + for _, r := range rows { + startFrom = r.name + entry := &filer.Entry{FullPath: util.NewFullPath(bucketsRoot, r.name)} + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil { + return bucketNames{}, fmt.Errorf("scan decode %s: %v", entry.FullPath, err) + } + if entry.IsDirectory() { + names.seen[r.name] = true + add(r.name) + } + } + if len(rows) < page { + break + } + } + + for _, name := range cachedBuckets { + add(name) + } + return names, nil +} + +// scanOneTable reads one table in primary-key pages and hands the entries +// strictly below dirPath to fn (the root itself is read separately). Key-value +// records share the default table; GenDirAndName stores the raw key bytes as +// their dirhash while entry rows hash the directory, so the mismatch +// identifies them exactly. +func (store *AbstractSqlStore) scanOneTable(ctx context.Context, tx *sql.Tx, t scanTable, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error { + type row struct { + dirHash int64 + name, dir string + data []byte + } + cursorHash, cursorName := int64(math.MinInt64), "" + firstPage := true + for { + var page []row + if t.cachedOnly && firstPage { + // A missing table can only be met by the first page read: once a + // row came back the tx holds the table against a drop. On + // PostgreSQL the failed query aborts the transaction, so the skip + // needs a savepoint to roll back to before the next table's read. + if _, err := tx.ExecContext(ctx, "SAVEPOINT seaweed_scan_bucket"); err != nil { + return fmt.Errorf("scan table %s: %v", t.table, err) + } + } + sqlText, args := store.GetSqlScanPage(t.table, cursorHash, cursorName, scanPageSize) + rows, err := tx.QueryContext(ctx, sqlText, args...) + if err != nil { + if t.cachedOnly && isMissingTableError(err) { + tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT seaweed_scan_bucket") + return nil // a cached-only name whose table was dropped meanwhile + } + return missingTableOr(err, t.table, fmt.Sprintf("scan table %s", t.table)) + } + firstPage = false + if err := func() error { + defer rows.Close() + for rows.Next() { + var r row + if err := rows.Scan(&r.dirHash, &r.name, &r.dir, &r.data); err != nil { + return fmt.Errorf("scan table %s: %v", t.table, err) + } + page = append(page, r) + } + return rows.Err() + }(); err != nil { + return err + } + + for _, r := range page { + cursorHash, cursorName = r.dirHash, r.name + if r.dirHash != util.HashStringToLong(r.dir) { + continue // a key-value record, see GenDirAndName + } + fullPath := util.NewFullPath(t.base+r.dir, r.name) + if !isStrictlyBelow(fullPath, dirPath) { + continue + } + entry := &filer.Entry{FullPath: fullPath} + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil { + return fmt.Errorf("scan decode %s: %v", fullPath, err) + } + if err := fn(ctx, entry); err != nil { + return err + } + } + if len(page) < scanPageSize { + return nil + } + } +} + +// missingTableOr names the condition when a bucket's table is not there. The +// scan fails closed: from inside the snapshot a table that was never created +// cannot be told from one dropped a moment ago, whose rows the snapshot still +// counts, and treating either as empty would let fsck purge referenced chunks. +func missingTableOr(err error, table, what string) error { + if table != DEFAULT_TABLE && isMissingTableError(err) { + return fmt.Errorf("bucket %s has a directory entry but no table (never created, or dropped during the scan): %v", table, err) + } + return fmt.Errorf("%s: %v", what, err) +} + +// isMissingTableError recognizes a missing table by message, as InsertEntry +// recognizes duplicates: PostgreSQL `relation "x" does not exist` (SQLSTATE +// 42P01), MySQL error 1146 (SQLSTATE 42S02), SQLite "no such table". +func isMissingTableError(err error) bool { + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "42p01") || + (strings.Contains(msg, "relation \"") && strings.Contains(msg, "does not exist")) || + strings.Contains(msg, "42s02") || + strings.Contains(msg, "error 1146") || + strings.Contains(msg, "no such table") +} + +// isStrictlyBelow reports whether p is a descendant of dir (not dir itself). +func isStrictlyBelow(p, dir util.FullPath) bool { + if dir == "/" { + return p != "/" + } + return strings.HasPrefix(string(p), string(dir)+"/") +} diff --git a/weed/filer/abstract_sql/abstract_sql_store_scan_test.go b/weed/filer/abstract_sql/abstract_sql_store_scan_test.go new file mode 100644 index 0000000000..e9c7cde6b0 --- /dev/null +++ b/weed/filer/abstract_sql/abstract_sql_store_scan_test.go @@ -0,0 +1,51 @@ +package abstract_sql + +import ( + "errors" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func TestBucketTableFor(t *testing.T) { + cases := map[string]string{ + "/": "", + "/buckets": "", + "/buckets/": "", + "/buckets/mybucket": "mybucket", + "/buckets/mybucket/a/b": "mybucket", + "/buckets/.system/owners": "", + "/buckets/Invalid_Bucket/x": "", + "/topics/.system/log": "", + "/bucketsfoo/mybucket": "", + } + for path, want := range cases { + if got := bucketTableFor(util.FullPath(path)); got != want { + t.Errorf("bucketTableFor(%s) = %q, want %q", path, got, want) + } + } +} + +// Only a missing table may be skipped; any other schema error must fail the +// scan, or a bucket's live chunks would be reported as orphans. +func TestIsMissingTableError(t *testing.T) { + for _, msg := range []string{ + `ERROR: relation "ghost" does not exist (SQLSTATE 42P01)`, + `Error 1146 (42S02): Table 'seaweedfs.ghost' doesn't exist`, + `SQL logic error: no such table: ghost (1)`, + } { + if !isMissingTableError(errors.New(msg)) { + t.Errorf("not recognized as a missing table: %s", msg) + } + } + for _, msg := range []string{ + `ERROR: column "directory" does not exist (SQLSTATE 42703)`, + `Error 1054 (42S22): Unknown column 'directory' in 'field list'`, + `SQL logic error: no such column: directory (1)`, + "connection refused", + } { + if isMissingTableError(errors.New(msg)) { + t.Errorf("wrongly taken for a missing table: %s", msg) + } + } +} diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 72f85f7951..748ef0df80 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -416,6 +416,28 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error { return f.Store.RollbackTransaction(ctx) } +type syncWriteKey struct{} + +// WithSyncWrite marks the entry writes under ctx durable-on-ack: a store that +// buffers writes by default must flush them before returning. +func WithSyncWrite(ctx context.Context) context.Context { + return context.WithValue(ctx, syncWriteKey{}, true) +} + +// SyncWrite reports whether ctx carries a durability request for entry writes. +func SyncWrite(ctx context.Context) bool { + return ctx.Value(syncWriteKey{}) != nil +} + +// syncWriteContext flags ctx when the storage rule on p asks for fsync, so the +// store write is held to the same durability as the data path. +func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context { + if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync { + return WithSyncWrite(ctx) + } + return ctx +} + // CreateEntry creates or replaces an entry. When existing is non-nil the caller // has already fetched the current entry at this path under a path lock, and it // is reused instead of looking the store up again; pass nil to have CreateEntry @@ -426,6 +448,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, return nil } + ctx = f.syncWriteContext(ctx, entry.FullPath) + if entry.FullPath.IsLongerFileName(maxFilenameLength) { return filer_pb.ErrEntryNameTooLong } @@ -631,6 +655,7 @@ func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath) // deleted with, so a restore cannot hand back a directory more permissive than the one // it replaces - including when someone else already put it back with wider ones. func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error { + ctx = f.syncWriteContext(ctx, dirPath) existing, err := f.FindEntry(ctx, dirPath) if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { return err @@ -680,6 +705,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, } func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) { + ctx = f.syncWriteContext(ctx, entry.FullPath) if oldEntry != nil { entry.Attr.Crtime = oldEntry.Attr.Crtime if oldEntry.Attr.Inode != 0 { diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 6f7949a615..899d337c05 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -122,6 +122,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR return nil } + ctx = f.syncWriteContext(ctx, p) + entry, findErr := f.FindEntry(ctx, p) if findErr != nil { return findErr diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index 9689436081..277903249a 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -44,6 +44,22 @@ type FilerStore interface { Shutdown() } +// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before +// any entry is visited, when the store cannot serve the scan. +var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store") + +// EntryScanner is an optional FilerStore capability: visit every entry at and +// below dirPath, in any order, from one consistent snapshot of the store. +// +// A directory walk sees each directory at a different time, so an entry +// renamed from a directory not yet listed into one already listed is never +// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it +// exactly once. The context handed to fn carries the snapshot, so store reads +// made with it (hard-link records) see the same instant. +type EntryScanner interface { + ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error +} + type BucketAware interface { OnBucketCreation(bucket string) OnBucketDeletion(bucket string) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 5ce0ff3d37..897c70152a 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -2,6 +2,7 @@ package filer import ( "context" + "errors" "fmt" "io" "math" @@ -19,6 +20,7 @@ import ( var ( _ = VirtualFilerStore(&FilerStoreWrapper{}) _ = Debuggable(&FilerStoreWrapper{}) + _ = EntryScanner(&FilerStoreWrapper{}) ) type VirtualFilerStore interface { @@ -402,6 +404,33 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return lastFileName, err } +// ScanEntries delegates to the default store. With path-specific stores +// configured, entries below dirPath could live in several stores and one +// snapshot per store is not one snapshot, so the scan is declined. Hard links +// are resolved inside the snapshot; a store error fails the scan (the per-path +// copy can be stale), a missing record falls back to it as listings do. +func (fsw *FilerStoreWrapper) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error { + if fsw.hasPathSpecificStore { + return ErrUnsupportedEntryScan + } + scanner, ok := fsw.defaultStore.(EntryScanner) + if !ok { + return ErrUnsupportedEntryScan + } + stats.FilerStoreCounter.WithLabelValues(fsw.defaultStore.GetName(), "scan").Inc() + start := time.Now() + defer func() { + stats.FilerStoreHistogram.WithLabelValues(fsw.defaultStore.GetName(), "scan").Observe(time.Since(start).Seconds()) + }() + return scanner.ScanEntries(ctx, dirPath, func(scanCtx context.Context, entry *Entry) error { + if err := fsw.maybeReadHardLink(scanCtx, entry); err != nil && !errors.Is(err, ErrKvNotFound) { + return fmt.Errorf("resolve hard link of %s: %w", entry.FullPath, err) + } + filer_pb.AfterEntryDeserialization(entry.GetChunks()) + return fn(scanCtx, entry) + }) +} + func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { actualStore := fsw.getActualStore(dirPath + "/") diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index a6d699d8be..606166da90 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -79,6 +79,15 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error { return nil } +var syncWriteOptions = &opt.WriteOptions{Sync: true} + +func writeOptions(ctx context.Context) *opt.WriteOptions { + if filer.SyncWrite(ctx) { + return syncWriteOptions + } + return nil +} + func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { key := genKey(entry.DirAndName()) @@ -91,7 +100,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) value = weed_util.MaybeGzipData(value) } - err = store.db.Put(key, value, nil) + err = store.db.Put(key, value, writeOptions(ctx)) if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -168,7 +177,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { key := genKey(fullpath.DirAndName()) - err = store.db.Delete(key, nil) + err = store.db.Delete(key, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -195,7 +204,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } iter.Release() - err = store.db.Write(batch, nil) + err = store.db.Write(batch, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index c3bb2ac55b..5c05033b99 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -6,6 +6,7 @@ import ( "crypto/md5" "fmt" "io" + "math" "os" "github.com/syndtr/goleveldb/leveldb" @@ -28,6 +29,11 @@ type LevelDB2Store struct { dbs []*leveldb.DB dbCount int ReadOnly bool + + blockCacheSizeMB int + writeBufferSizeMB int + blockSizeKB int + compactionTableSizeMB int } func (store *LevelDB2Store) GetName() string { @@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string { func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) { dir := configuration.GetString(prefix + "dir") + store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb") + store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb") + store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb") + store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb") + if err := store.validateSizeOptions(); err != nil { + return err + } return store.initialize(dir, 8) } +// validateSizeOptions rejects sizes too large to convert to bytes without +// overflowing int on this platform. +func (store *LevelDB2Store) validateSizeOptions() error { + for name, kv := range map[string][2]int{ + "block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20}, + "write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20}, + "block_size_kb": {store.blockSizeKB, 1 << 10}, + "compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20}, + } { + if kv[0] > math.MaxInt/kv[1] { + return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0]) + } + } + return nil +} + +func (store *LevelDB2Store) buildOptions() *opt.Options { + opts := &opt.Options{ + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + Filter: filter.NewBloomFilter(8), // false positive rate 0.02 + ReadOnly: store.ReadOnly, + } + if store.blockCacheSizeMB > 0 { + opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024 + } + if store.writeBufferSizeMB > 0 { + opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024 + } + if store.blockSizeKB > 0 { + opts.BlockSize = store.blockSizeKB * 1024 + } + if store.compactionTableSizeMB > 0 { + opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024 + } + return opts +} + func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { glog.Infof("filer store leveldb2 dir: %s", dir) os.MkdirAll(dir, 0755) @@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) } - opts := &opt.Options{ - BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB - Filter: filter.NewBloomFilter(8), // false positive rate 0.02 - ReadOnly: store.ReadOnly, - } + opts := store.buildOptions() for d := 0; d < dbCount; d++ { dbFolder := fmt.Sprintf("%s/%02d", dir, d) @@ -81,6 +127,15 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error { return nil } +var syncWriteOptions = &opt.WriteOptions{Sync: true} + +func writeOptions(ctx context.Context) *opt.WriteOptions { + if filer.SyncWrite(ctx) { + return syncWriteOptions + } + return nil +} + func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.DirAndName() key, partitionId := genKey(dir, name, store.dbCount) @@ -94,7 +149,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) value = weed_util.MaybeGzipData(value) } - err = store.dbs[partitionId].Put(key, value, nil) + err = store.dbs[partitionId].Put(key, value, writeOptions(ctx)) if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -140,7 +195,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util. dir, name := fullpath.DirAndName() key, partitionId := genKey(dir, name, store.dbCount) - err = store.dbs[partitionId].Delete(key, nil) + err = store.dbs[partitionId].Delete(key, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -167,7 +222,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } iter.Release() - err = store.dbs[partitionId].Write(batch, nil) + err = store.dbs[partitionId].Write(batch, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index 5c6a3a4d71..9651711d92 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -2,6 +2,8 @@ package leveldb import ( "context" + "math" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) { } +type fakeConfiguration map[string]interface{} + +func (c fakeConfiguration) GetString(key string) string { + value, _ := c[key].(string) + return value +} + +func (c fakeConfiguration) GetBool(key string) bool { + value, _ := c[key].(bool) + return value +} + +func (c fakeConfiguration) GetInt(key string) int { + value, _ := c[key].(int) + return value +} + +func (c fakeConfiguration) GetStringSlice(key string) []string { + value, _ := c[key].([]string) + return value +} + +func (c fakeConfiguration) SetDefault(key string, value interface{}) { + if _, found := c[key]; !found { + c[key] = value + } +} + +func TestInitializeOptions(t *testing.T) { + store := &LevelDB2Store{} + if err := store.Initialize(fakeConfiguration{ + "leveldb2.dir": t.TempDir(), + "leveldb2.block_cache_size_mb": 64, + "leveldb2.write_buffer_size_mb": 32, + "leveldb2.block_size_kb": 16, + "leveldb2.compaction_table_size_mb": 8, + }, "leveldb2."); err != nil { + t.Fatalf("initialize: %v", err) + } + defer store.Shutdown() + + opts := store.buildOptions() + if opts.BlockCacheCapacity != 64*1024*1024 { + t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity) + } + if opts.WriteBuffer != 32*1024*1024 { + t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer) + } + if opts.BlockSize != 16*1024 { + t.Errorf("block size %d, want 16KiB", opts.BlockSize) + } + if opts.CompactionTableSize != 8*1024*1024 { + t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize) + } +} + +func TestInitializeDefaultOptions(t *testing.T) { + store := &LevelDB2Store{} + if err := store.Initialize(fakeConfiguration{ + "leveldb2.dir": t.TempDir(), + "leveldb2.block_cache_size_mb": -1, + "leveldb2.write_buffer_size_mb": 0, + }, "leveldb2."); err != nil { + t.Fatalf("initialize: %v", err) + } + defer store.Shutdown() + + opts := store.buildOptions() + if opts.BlockCacheCapacity != 32*1024*1024 { + t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity) + } + if opts.WriteBuffer != 16*1024*1024 { + t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer) + } +} + +func TestInitializeRejectsOversizedOption(t *testing.T) { + store := &LevelDB2Store{} + err := store.Initialize(fakeConfiguration{ + "leveldb2.dir": t.TempDir(), + "leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1, + }, "leveldb2.") + if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") { + t.Fatalf("expected overflow rejection, got %v", err) + } +} + func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil) dir := t.TempDir() diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 564d6eaddd..029d902715 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -6,6 +6,7 @@ import ( "crypto/md5" "fmt" "io" + "math" "os" "strings" "sync" @@ -35,6 +36,11 @@ type LevelDB3Store struct { dbs map[string]*leveldb.DB dbsLock sync.RWMutex ReadOnly bool + + blockCacheSizeMB int + writeBufferSizeMB int + blockSizeKB int + compactionTableSizeMB int } func (store *LevelDB3Store) GetName() string { @@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string { func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) { dir := configuration.GetString(prefix + "dir") + store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb") + store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb") + store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb") + store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb") + if err := store.validateSizeOptions(); err != nil { + return err + } return store.initialize(dir) } +// validateSizeOptions rejects sizes too large to convert to bytes without +// overflowing int on this platform. +func (store *LevelDB3Store) validateSizeOptions() error { + for name, kv := range map[string][2]int{ + "block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20}, + "write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20}, + "block_size_kb": {store.blockSizeKB, 1 << 10}, + "compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20}, + } { + if kv[0] > math.MaxInt/kv[1] { + return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0]) + } + } + return nil +} + +func (store *LevelDB3Store) buildOptions(name string) *opt.Options { + opts := &opt.Options{ + BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB + WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB + Filter: filter.NewBloomFilter(8), + ReadOnly: store.ReadOnly, + } + if name != DEFAULT { + opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB + opts.WriteBuffer = 8 * 1024 * 1024 // default value is 4MiB + } + if store.blockCacheSizeMB > 0 { + opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024 + } + if store.writeBufferSizeMB > 0 { + opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024 + } + if store.blockSizeKB > 0 { + opts.BlockSize = store.blockSizeKB * 1024 + } + if store.compactionTableSizeMB > 0 { + opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024 + } + return opts +} + func (store *LevelDB3Store) initialize(dir string) (err error) { glog.Infof("filer store leveldb3 dir: %s", dir) os.MkdirAll(dir, 0755) @@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) { } func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { - bloom := filter.NewBloomFilter(8) // false positive rate 0.02 - opts := &opt.Options{ - BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB - Filter: bloom, - ReadOnly: store.ReadOnly, - } - if name != DEFAULT { - opts = &opt.Options{ - BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB - WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB - Filter: bloom, - ReadOnly: store.ReadOnly, - } - } + opts := store.buildOptions(name) dbFolder := fmt.Sprintf("%s/%s", store.dir, name) os.MkdirAll(dbFolder, 0755) @@ -180,6 +221,15 @@ func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error { return nil } +var syncWriteOptions = &opt.WriteOptions{Sync: true} + +func writeOptions(ctx context.Context) *opt.WriteOptions { + if filer.SyncWrite(ctx) { + return syncWriteOptions + } + return nil +} + func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { db, _, shortPath, err := store.findDB(entry.FullPath, false) @@ -199,7 +249,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) value = weed_util.MaybeGzipData(value) } - err = db.Put(key, value, nil) + err = db.Put(key, value, writeOptions(ctx)) if err != nil { return fmt.Errorf("persisting %s : %v", entry.FullPath, err) @@ -257,7 +307,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util. dir, name := shortPath.DirAndName() key := genKey(dir, name) - err = db.Delete(key, nil) + err = db.Delete(key, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -298,7 +348,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w } iter.Release() - err = db.Write(batch, nil) + err = db.Write(batch, writeOptions(ctx)) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index 8a3cd2d82e..62896a299f 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -2,6 +2,8 @@ package leveldb import ( "context" + "math" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) { } +type fakeConfiguration map[string]interface{} + +func (c fakeConfiguration) GetString(key string) string { + value, _ := c[key].(string) + return value +} + +func (c fakeConfiguration) GetBool(key string) bool { + value, _ := c[key].(bool) + return value +} + +func (c fakeConfiguration) GetInt(key string) int { + value, _ := c[key].(int) + return value +} + +func (c fakeConfiguration) GetStringSlice(key string) []string { + value, _ := c[key].([]string) + return value +} + +func (c fakeConfiguration) SetDefault(key string, value interface{}) { + if _, found := c[key]; !found { + c[key] = value + } +} + +func TestInitializeOptions(t *testing.T) { + store := &LevelDB3Store{} + if err := store.Initialize(fakeConfiguration{ + "leveldb3.dir": t.TempDir(), + "leveldb3.block_cache_size_mb": 64, + "leveldb3.write_buffer_size_mb": 32, + "leveldb3.block_size_kb": 16, + "leveldb3.compaction_table_size_mb": 8, + }, "leveldb3."); err != nil { + t.Fatalf("initialize: %v", err) + } + defer store.Shutdown() + + for _, name := range []string{DEFAULT, "bucket-a"} { + opts := store.buildOptions(name) + if opts.BlockCacheCapacity != 64*1024*1024 { + t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity) + } + if opts.WriteBuffer != 32*1024*1024 { + t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer) + } + if opts.BlockSize != 16*1024 { + t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize) + } + if opts.CompactionTableSize != 8*1024*1024 { + t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize) + } + } +} + +func TestInitializeDefaultOptions(t *testing.T) { + store := &LevelDB3Store{} + if err := store.Initialize(fakeConfiguration{ + "leveldb3.dir": t.TempDir(), + "leveldb3.block_cache_size_mb": -1, + "leveldb3.write_buffer_size_mb": 0, + }, "leveldb3."); err != nil { + t.Fatalf("initialize: %v", err) + } + defer store.Shutdown() + + opts := store.buildOptions(DEFAULT) + if opts.BlockCacheCapacity != 32*1024*1024 { + t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity) + } + if opts.WriteBuffer != 16*1024*1024 { + t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer) + } + + bucketOpts := store.buildOptions("bucket-a") + if bucketOpts.BlockCacheCapacity != 16*1024*1024 { + t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity) + } + if bucketOpts.WriteBuffer != 8*1024*1024 { + t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer) + } +} + +func TestInitializeRejectsOversizedOption(t *testing.T) { + store := &LevelDB3Store{} + err := store.Initialize(fakeConfiguration{ + "leveldb3.dir": t.TempDir(), + "leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1, + }, "leveldb3.") + if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") { + t.Fatalf("expected overflow rejection, got %v", err) + } +} + func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil) dir := t.TempDir() diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index 049ea685a0..3566e4e99c 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -62,6 +62,15 @@ func (gen *SqlGenMysql) nameExpr() string { return "`name`" } +// GetSqlScanPage spells the cursor as two primary-key ranges: MySQL 5.7 does +// not turn a row-value comparison into a range seek. The name keeps the +// column's own collation (BINARY would filesort every page); the scan promises +// no lexical order. SQLite overrides this method. +func (gen *SqlGenMysql) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) { + return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE `dirhash` > ? OR (`dirhash` = ? AND `name` > ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName), + []interface{}{dirhash, dirhash, name, limit} +} + func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string { name := gen.nameExpr() return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name) diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go index 9ff50d96a1..4ba99354cf 100644 --- a/weed/filer/postgres/postgres_sql_gen.go +++ b/weed/filer/postgres/postgres_sql_gen.go @@ -75,6 +75,14 @@ func (gen *SqlGenPostgres) nameExpr() string { return "name" } +// GetSqlScanPage uses a row-value comparison, one primary-key range seek on +// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would +// sort every page); the scan promises no lexical order. +func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) { + return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName), + []interface{}{dirhash, name, limit} +} + func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string { name := gen.nameExpr() return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name) diff --git a/weed/filer/sqlite/sqlite_store.go b/weed/filer/sqlite/sqlite_store.go index 31d696ca73..7a45ef83ca 100644 --- a/weed/filer/sqlite/sqlite_store.go +++ b/weed/filer/sqlite/sqlite_store.go @@ -30,6 +30,31 @@ func (store *SqliteStore) GetName() string { return "sqlite" } +// sqliteSqlGen is the MySQL generator with SQLite's own page cursor: SQLite +// seeks a row-value comparison on the primary key but scans the OR form. +type sqliteSqlGen struct { + *mysql.SqlGenMysql +} + +func (gen sqliteSqlGen) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) { + return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE (`dirhash`, `name`) > (?, ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName), + []interface{}{dirhash, name, limit} +} + +// ScanEntries offers the snapshot scan only in WAL mode: with a rollback +// journal a long read transaction holds a shared lock and writers fail after +// the busy timeout. +func (store *SqliteStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error { + var mode string + if err := store.DB.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil { + return fmt.Errorf("%w: journal_mode: %v", filer.ErrUnsupportedEntryScan, err) + } + if !strings.EqualFold(mode, "wal") { + return fmt.Errorf("%w: sqlite journal_mode is %q, the scan needs wal", filer.ErrUnsupportedEntryScan, mode) + } + return store.AbstractSqlStore.ScanEntries(ctx, dirPath, fn) +} + func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) { dbFile := configuration.GetString(prefix + "dbFile") createTable := `CREATE TABLE IF NOT EXISTS "%s" ( @@ -69,11 +94,11 @@ func sqliteDSN(dbFile string) string { func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) { store.SupportBucketTable = true - store.SqlGenerator = &mysql.SqlGenMysql{ + store.SqlGenerator = sqliteSqlGen{&mysql.SqlGenMysql{ CreateTableSqlTemplate: createTable, DropTableSqlTemplate: "drop table if exists `%s`", UpsertQueryTemplate: upsertQuery, - } + }} dsn := sqliteDSN(dbFile) @@ -94,6 +119,11 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e }, 1, 1, 0); err != nil { return err } + // The main pool is one connection, which a long scan must not hold. + if store.ScanDB, err = sql.Open("sqlite", dsn); err != nil { + return fmt.Errorf("open scan connection to %s: %v", dbFile, err) + } + store.ScanDB.SetMaxOpenConns(1) if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) diff --git a/weed/filer/sqlite/sqlite_store_scan_test.go b/weed/filer/sqlite/sqlite_store_scan_test.go new file mode 100644 index 0000000000..b859229171 --- /dev/null +++ b/weed/filer/sqlite/sqlite_store_scan_test.go @@ -0,0 +1,288 @@ +//go:build (linux || darwin || windows) && sqlite + +package sqlite + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// A rename committed while a scan is running is seen under exactly one of its +// names. The scan reads from one snapshot, so unlike a directory walk it can +// neither miss the entry when it moves into a directory already listed, nor +// count it twice when it moves into one not listed yet. +func TestScanEntriesSeesConcurrentRenameExactlyOnce(t *testing.T) { + dbFile := filepath.Join(t.TempDir(), "filer.db") + reader := &SqliteStore{} + if err := reader.initialize(dbFile, createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer reader.Shutdown() + // WAL lets the writer commit while the scan's read transaction is open. + if _, err := reader.DB.Exec("PRAGMA journal_mode=WAL"); err != nil { + t.Fatal(err) + } + writer := &SqliteStore{} + if err := writer.initialize(dbFile, createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer writer.Shutdown() + + ctx := context.Background() + mk := func(p string) *filer.Entry { + return &filer.Entry{FullPath: util.FullPath(p), Attr: filer.Attr{Mode: 0644, Mtime: time.Now(), Crtime: time.Now()}} + } + for _, p := range []string{"/ckpt/a", "/ckpt/b", "/ckpt/tmp/x"} { + if err := reader.InsertEntry(ctx, mk(p)); err != nil { + t.Fatal(err) + } + } + + defer func(n int) { abstract_sql.SetScanPageSizeForTest(n) }(abstract_sql.SetScanPageSizeForTest(2)) + + seen := map[string]int{} + renamed := false + err := reader.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error { + seen[string(e.FullPath)]++ + if renamed { + return nil + } + renamed = true + // /ckpt/tmp/x -> /ckpt/x, committed by another connection mid-scan. + if err := writer.InsertEntry(ctx, mk("/ckpt/x")); err != nil { + return err + } + return writer.DeleteEntry(ctx, util.FullPath("/ckpt/tmp/x")) + }) + if err != nil { + t.Fatal(err) + } + if !renamed { + t.Fatal("scan yielded nothing") + } + if seen["/ckpt/a"] != 1 || seen["/ckpt/b"] != 1 { + t.Errorf("untouched entries seen %d and %d times, want once each", seen["/ckpt/a"], seen["/ckpt/b"]) + } + if old, new := seen["/ckpt/tmp/x"], seen["/ckpt/x"]; old+new != 1 { + t.Errorf("renamed entry seen %d time(s) under the old name and %d under the new, want exactly one in total", old, new) + } else if old != 1 { + t.Errorf("the rename committed after the snapshot was taken, so the scan must show the old name") + } + if _, err := reader.FindEntry(ctx, util.FullPath("/ckpt/x")); err != nil { + t.Errorf("once the scan is over the new name must be visible: %v", err) + } +} + +// Rows that are not entries, key-value records and a regular file under +// /buckets with a bucket-like name, are stepped over; a scan rooted at a bucket +// yields the bucket's own row (default table) and its contents (bucket table). +func TestScanEntriesRowsAndBucketTables(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + now := time.Now() + // The filer creates bucket tables on bucket creation; the store layer alone does not. + if err := store.CreateTable(ctx, "realbucket"); err != nil { + t.Fatal(err) + } + for _, e := range []*filer.Entry{ + {FullPath: "/buckets", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}, + {FullPath: "/buckets/realbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}, + {FullPath: "/buckets/realbucket/f1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}}, + {FullPath: "/buckets/readme", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}}, + {FullPath: "/topics/t1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}}, + } { + if err := store.InsertEntry(ctx, e); err != nil { + t.Fatal(err) + } + } + for _, key := range []string{"filer.store.id", "/looks/like/a/path", "x"} { + if err := store.KvPut(ctx, []byte(key), []byte("opaque value, not an Entry")); err != nil { + t.Fatal(err) + } + } + + collect := func(root string) []string { + var got []string + if err := store.ScanEntries(ctx, util.FullPath(root), func(_ context.Context, e *filer.Entry) error { + got = append(got, string(e.FullPath)) + return nil + }); err != nil { + t.Fatalf("scan %s: %v", root, err) + } + sort.Strings(got) + return got + } + if got, want := collect("/"), []string{"/buckets", "/buckets/readme", "/buckets/realbucket", "/buckets/realbucket/f1", "/topics/t1"}; !reflect.DeepEqual(got, want) { + t.Errorf("scan /: got %v, want %v", got, want) + } + if got, want := collect("/buckets/realbucket"), []string{"/buckets/realbucket", "/buckets/realbucket/f1"}; !reflect.DeepEqual(got, want) { + t.Errorf("scan /buckets/realbucket: got %v, want %v", got, want) + } + + // A bucket directory whose table does not exist fails the scan: from the + // snapshot it cannot be told from a table dropped a moment ago. + if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/buckets/ghostbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}); err != nil { + t.Fatal(err) + } + err := store.ScanEntries(ctx, "/", func(context.Context, *filer.Entry) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "ghostbucket") { + t.Fatalf("scan over a bucket without a table must fail naming it, got %v", err) + } +} + +// Without WAL the store declines the scan rather than block writers for its +// whole duration. +func TestScanEntriesDeclinedWithoutWAL(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }) + if !errors.Is(err, filer.ErrUnsupportedEntryScan) { + t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err) + } +} + +// A store whose pool is one connection, and no scan connection of its own, +// declines rather than hold that connection for the whole scan. +func TestScanEntriesDeclinedOnSingleConnectionPool(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil { + t.Fatal(err) + } + store.ScanDB.Close() + store.ScanDB = nil + err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }) + if !errors.Is(err, filer.ErrUnsupportedEntryScan) { + t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err) + } +} + +// A hard-linked entry's chunks live in the shared key-value record, not in +// its path row; the wrapper resolves them from inside the snapshot, on the +// single connection the scan holds. +func TestScanEntriesResolvesHardLinks(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil { + t.Fatal(err) + } + wrapper := filer.NewFilerStoreWrapper(store) + ctx := context.Background() + now := time.Now() + linked := &filer.Entry{ + FullPath: "/buckets/hlbucket/f0001", + Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}, + HardLinkId: filer.HardLinkId("hardlink-00000001"), + HardLinkCounter: 2, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 4096}}, + } + if err := wrapper.InsertEntry(ctx, linked); err != nil { + t.Fatal(err) + } + + var got *filer.Entry + if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error { + if e.FullPath == linked.FullPath { + got = e + } + return nil + }); err != nil { + t.Fatal(err) + } + if got == nil || len(got.GetChunks()) != 1 || got.GetChunks()[0].FileId != "3,01637037d6" || got.HardLinkCounter != 2 { + t.Fatalf("hard-linked entry not resolved from its key-value record: %+v", got) + } + + // Once the record is gone the path row is all there is; the scan hands it + // over instead of failing on that entry every time it runs. + if err := store.KvDelete(ctx, []byte(linked.HardLinkId)); err != nil { + t.Fatal(err) + } + got = nil + if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error { + if e.FullPath == linked.FullPath { + got = e + } + return nil + }); err != nil { + t.Fatalf("a missing hard-link record must not fail the scan: %v", err) + } + if got == nil { + t.Fatal("entry with a missing hard-link record not handed over") + } +} + +// The keyset page must be a primary-key seek; an OR-shaped cursor predicate +// makes SQLite scan the table from the start for every page. +func TestScanPageQuerySeeksPrimaryKey(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + sqlText, args := store.GetSqlScanPage("filemeta", int64(-1), "", 10) + rows, err := store.DB.Query("EXPLAIN QUERY PLAN "+sqlText, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + joined := strings.Join(plan, " | ") + if !strings.Contains(joined, "SEARCH") || strings.Contains(joined, "SCAN") { + t.Fatalf("page query does not seek the primary key: %s", joined) + } +} + +// A bucket name only the process cache remembers may have its table dropped +// meanwhile; the scan skips it instead of failing. +func TestScanEntriesSkipsCachedOnlyBucketTable(t *testing.T) { + store := &SqliteStore{} + if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil { + t.Fatal(err) + } + store.OnBucketCreation("gonebucket") // the table is never created + if err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }); err != nil { + t.Fatalf("cached-only bucket with a dropped table must not fail the scan: %v", err) + } +} diff --git a/weed/iam/helpers.go b/weed/iam/helpers.go index 628fa9551d..6d006478bb 100644 --- a/weed/iam/helpers.go +++ b/weed/iam/helpers.go @@ -98,11 +98,13 @@ var fineGrainedActionMap = map[string]string{ // prefixed (s3:Action) and unprefixed (Action) variants in init(). var baseS3ActionMap = map[string]string{ // Object operations - "DeleteObject": s3_constants.ACTION_WRITE, - "PutObject": s3_constants.ACTION_WRITE, - "GetObject": s3_constants.ACTION_READ, - "DeleteObjectVersion": s3_constants.ACTION_WRITE, - "GetObjectVersion": s3_constants.ACTION_READ, + "DeleteObject": s3_constants.ACTION_WRITE, + "PutObject": s3_constants.ACTION_WRITE, + "GetObject": s3_constants.ACTION_READ, + "DeleteObjectVersion": s3_constants.ACTION_WRITE, + "GetObjectVersion": s3_constants.ACTION_READ, + "GetObjectAttributes": s3_constants.ACTION_READ, + "GetObjectVersionAttributes": s3_constants.ACTION_READ, // Tagging operations "GetObjectTagging": s3_constants.ACTION_TAGGING, "GetObjectVersionTagging": s3_constants.ACTION_TAGGING, @@ -115,6 +117,7 @@ var baseS3ActionMap = map[string]string{ "DeleteBucketTagging": s3_constants.ACTION_TAGGING, // ACL operations "PutObjectAcl": s3_constants.ACTION_WRITE_ACP, + "PutObjectVersionAcl": s3_constants.ACTION_WRITE_ACP, "GetObjectAcl": s3_constants.ACTION_READ_ACP, "GetObjectVersionAcl": s3_constants.ACTION_READ_ACP, "PutBucketAcl": s3_constants.ACTION_WRITE_ACP, diff --git a/weed/iam/integration/iam_manager.go b/weed/iam/integration/iam_manager.go index b3af9c7b36..ece1a54927 100644 --- a/weed/iam/integration/iam_manager.go +++ b/weed/iam/integration/iam_manager.go @@ -53,7 +53,13 @@ type IAMManager struct { oidcRetryAgain bool // oidcRefreshMu serializes refreshes from reading the store to handing // STS the result, so an older snapshot cannot replace a newer one. - oidcRefreshMu sync.Mutex + oidcRefreshMu sync.Mutex + // oidcManagedProviders caches providers refreshOIDCProvidersFrom built, + // keyed by record identity, so an unchanged record reuses its provider: + // rebuilding would drop the provider's consumed-token (jti) replay cache + // and let an already-used token be presented again. Guarded by + // oidcRefreshMu. + oidcManagedProviders map[string]*oidcManagedProvider oidcAuditSink OIDCProviderAuditSink revocationStore SessionRevocationStore filerAddressProvider func() string // Function to get current filer address @@ -943,14 +949,65 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro return fmt.Errorf("list OIDC providers: the store was replaced during the refresh") } byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records)) + if m.oidcManagedProviders == nil { + m.oidcManagedProviders = make(map[string]*oidcManagedProvider) + } + seen := make(map[string]struct{}, len(records)) + // Records for one issuer with identical validation settings share a single + // provider — and with it the consumed-token (jti) replay store — so a token + // spent through one account's record stays spent through the other's too. + sharedProviders := make(map[string]*oidc.OIDCProvider) for _, rec := range records { if rec == nil || rec.URL == "" { continue } - provider, err := buildOIDCProviderFromRecord(rec) - if err != nil { - glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err) - continue + key := rec.ARN + if key == "" { + key = rec.URL + "\x00" + rec.AccountID + } + seen[key] = struct{}{} + fingerprint := oidcProviderFingerprint(rec) + configFingerprint := oidcProviderConfigFingerprint(rec) + var provider *oidc.OIDCProvider + cached := m.oidcManagedProviders[key] + if shared, ok := sharedProviders[configFingerprint]; ok { + provider = shared + if cached != nil && cached.provider != nil && cached.provider != shared { + // The record's own provider is superseded; its consumed + // tokens move to the shared store so none reopen for replay. + shared.MergeReplayStateFrom(cached.provider) + } + m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider} + if cached != nil && cached.provider != nil && cached.provider != shared && !m.oidcProviderStillReferenced(cached.provider) { + if err := cached.provider.Shutdown(ctx); err != nil { + glog.Warningf("shutdown superseded OIDC provider %s: %v", rec.ARN, err) + } + } + } else if cached != nil && cached.fingerprint == fingerprint { + provider = cached.provider + sharedProviders[configFingerprint] = provider + } else { + var err error + provider, err = buildOIDCProviderFromRecord(rec) + if err != nil { + delete(m.oidcManagedProviders, key) + glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err) + continue + } + if cached != nil && cached.provider != nil { + // The rebuild must keep rejecting tokens the replaced + // provider consumed; carry its replay store over, then stop + // its cleanup worker so only the new one sweeps the store — + // unless another record still references that provider. + provider.AdoptReplayStateFrom(cached.provider) + } + sharedProviders[configFingerprint] = provider + m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider} + if cached != nil && cached.provider != nil && !m.oidcProviderStillReferenced(cached.provider) { + if err := cached.provider.Shutdown(ctx); err != nil { + glog.Warningf("shutdown replaced OIDC provider %s: %v", rec.ARN, err) + } + } } // Multiple records may share an issuer when each is scoped to a // different account; STS picks the right one at validation time @@ -960,10 +1017,71 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro Provider: provider, }) } + for key, managed := range m.oidcManagedProviders { + if _, ok := seen[key]; !ok { + delete(m.oidcManagedProviders, key) + // A removed record's provider may be shared by records that are + // still present; shut it down only when nothing references it. + if managed.provider != nil && !m.oidcProviderStillReferenced(managed.provider) { + if err := managed.provider.Shutdown(ctx); err != nil { + glog.Warningf("shutdown removed OIDC provider %s: %v", key, err) + } + } + } + } m.stsService.SetIAMManagedOIDCProviders(byIssuer) return nil } +// oidcManagedProvider is a runtime provider together with the fingerprint of +// the record it was built from. +type oidcManagedProvider struct { + fingerprint string + provider *oidc.OIDCProvider +} + +// oidcProviderStillReferenced reports whether any managed record still points +// at the provider, so a shared provider is not shut down with a removed record. +func (m *IAMManager) oidcProviderStillReferenced(provider *oidc.OIDCProvider) bool { + for _, managed := range m.oidcManagedProviders { + if managed.provider == provider { + return true + } + } + return false +} + +// oidcProviderFingerprint identifies the record content a provider was built +// from, so a refresh rebuilds the provider only when that content changed. +func oidcProviderFingerprint(rec *OIDCProviderRecord) string { + key, _ := json.Marshal(struct { + ARN string + AccountID string + URL string + ClientIDs []string + Thumbprints []string + AllowedPrincipalTagKeys []string + PolicyClaim string + }{rec.ARN, rec.AccountID, rec.URL, rec.ClientIDs, rec.Thumbprints, rec.AllowedPrincipalTagKeys, rec.PolicyClaim}) + sum := sha256.Sum256(key) + return hex.EncodeToString(sum[:]) +} + +// oidcProviderConfigFingerprint identifies the validation settings a provider +// is built from — everything buildOIDCProviderFromRecord consumes except the +// record's name — so records that differ only in identity reuse one provider. +func oidcProviderConfigFingerprint(rec *OIDCProviderRecord) string { + key, _ := json.Marshal(&oidc.OIDCConfig{ + Issuer: rec.URL, + ClientIDs: rec.ClientIDs, + Thumbprints: rec.Thumbprints, + AllowedPrincipalTagKeys: rec.AllowedPrincipalTagKeys, + PolicyClaim: rec.PolicyClaim, + }) + sum := sha256.Sum256(key) + return hex.EncodeToString(sum[:]) +} + // buildOIDCProviderFromRecord turns a stored record into a runtime // OIDCProvider. The provider name is the ARN so re-registration is // idempotent and never collides with static-config entries. diff --git a/weed/iam/integration/oidc_provider_persist_test.go b/weed/iam/integration/oidc_provider_persist_test.go index 7190e2aa74..26a6f81250 100644 --- a/weed/iam/integration/oidc_provider_persist_test.go +++ b/weed/iam/integration/oidc_provider_persist_test.go @@ -559,3 +559,93 @@ func TestAFailureDuringARetryIsNotDropped(t *testing.T) { } assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran") } + +// A refresh must not rebuild a provider whose record is unchanged: rebuilding +// would drop the provider's consumed-token (jti) replay cache and let an +// already-used token be presented again. +func TestRefreshReusesProviderWhileRecordIsUnchanged(t *testing.T) { + ctx := context.Background() + store := NewMemoryOIDCProviderStore() + mgr := startServer(t, store) + + arn := arnOf(t, persistTestAPIIssuer) + require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{ + ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud"}, + })) + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + first := mgr.oidcManagedProviders[arn].provider + require.NotNil(t, first) + + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + assert.Same(t, first, mgr.oidcManagedProviders[arn].provider, + "an unchanged record was rebuilt, dropping its replay cache") + + // A changed record must rebuild so the new configuration applies. + require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{ + ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud", "aud2"}, + })) + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + rebuilt := mgr.oidcManagedProviders[arn].provider + require.NotNil(t, rebuilt) + assert.NotSame(t, first, rebuilt, "a changed record kept the old provider") + + // Deleting the record drops the provider from the cache and the STS map. + require.NoError(t, store.DeleteProvider(ctx, "", arn)) + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + assert.NotContains(t, mgr.oidcManagedProviders, arn) + assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer)) +} + +// stsKnowsIssuerForAccount is stsKnowsIssuer through the account-scoped +// lookup used for providers that have no global record. +func stsKnowsIssuerForAccount(t *testing.T, mgr *IAMManager, issuer, accountID string) bool { + t.Helper() + tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(), + }).SignedString([]byte("not-the-providers-key")) + require.NoError(t, err) + _, _, err = mgr.GetSTSService().ValidateWebIdentityTokenForAccount(context.Background(), tok, accountID) + require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer) + return !strings.Contains(err.Error(), "no identity provider registered") +} + +// Two records for one issuer with identical validation settings share a +// single provider — and with it the consumed-token (jti) replay store — so a +// token spent through one account's record stays spent through the other's. +// Removing one record of the pair must not shut the shared provider down. +func TestSameIssuerRecordsShareOneProvider(t *testing.T) { + ctx := context.Background() + store := NewMemoryOIDCProviderStore() + mgr := startServer(t, store) + + arnA := arnOf(t, persistTestAPIIssuer) + "-account-a" + arnB := arnOf(t, persistTestAPIIssuer) + "-account-b" + for _, rec := range []*OIDCProviderRecord{ + {ARN: arnA, URL: persistTestAPIIssuer, AccountID: "111111111111", ClientIDs: []string{"aud"}}, + {ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"aud"}}, + } { + require.NoError(t, store.StoreProvider(ctx, "", rec)) + } + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + + shared := mgr.oidcManagedProviders[arnA].provider + require.NotNil(t, shared) + assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider, + "identical records for one issuer built separate replay stores") + + // Removing one record of the pair keeps the shared provider alive. + require.NoError(t, store.DeleteProvider(ctx, "", arnA)) + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider, + "a removed sibling record shut down the shared provider") + assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222")) + + // The remaining record diverging rebuilds its own provider. + require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{ + ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"b-aud"}, + })) + require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store)) + assert.NotSame(t, shared, mgr.oidcManagedProviders[arnB].provider, + "a changed record kept the shared provider") + assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222")) +} diff --git a/weed/iam/oidc/mock_provider.go b/weed/iam/oidc/mock_provider.go index bc20a5f14f..5b840eafa7 100644 --- a/weed/iam/oidc/mock_provider.go +++ b/weed/iam/oidc/mock_provider.go @@ -75,6 +75,17 @@ func (m *MockOIDCProvider) Authenticate(ctx context.Context, token string) (*pro return identity, nil } +// AuthenticateSingleUse stays on the mock path: the embedded provider's +// method would bypass the mock's test data and demand a real JWKS. +func (m *MockOIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) { + return m.Authenticate(ctx, token) +} + +// ValidateTokenSingleUse stays on the mock path; see AuthenticateSingleUse. +func (m *MockOIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) { + return m.ValidateToken(ctx, token) +} + // ValidateToken validates tokens using test data func (m *MockOIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) { if !m.initialized { diff --git a/weed/iam/oidc/oidc_provider.go b/weed/iam/oidc/oidc_provider.go index 4f947cf2ba..c2ccf4ab8f 100644 --- a/weed/iam/oidc/oidc_provider.go +++ b/weed/iam/oidc/oidc_provider.go @@ -6,6 +6,7 @@ import ( "crypto/elliptic" "crypto/rsa" "crypto/sha1" + "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/base64" @@ -19,6 +20,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/golang-jwt/jwt/v5" @@ -27,7 +29,44 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" ) -// OIDCProvider implements OpenID Connect authentication +const ( + // clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519) + clockSkewTolerance = 5 * time.Minute + // defaultJTICacheSize is the default maximum number of JTI entries to cache + defaultJTICacheSize = 10000 + // defaultCleanupInterval is how often to scan for expired JTI entries + defaultCleanupInterval = 5 * time.Minute + // eagerCleanupMinInterval rate limits opportunistic sweeps near capacity + eagerCleanupMinInterval = 5 * time.Second + // tokenFingerprintPrefix marks replay-cache keys derived from the token's + // own fingerprint, used when the token carries no usable jti claim + tokenFingerprintPrefix = "fp:" +) + +// jtiEntry represents a cached JTI with metadata +type jtiEntry struct { + subject string + expiresAt time.Time + // uncounted marks an entry merged in past the capacity bound: it keeps + // rejecting replays but does not charge jtiCount, so merged excess can + // neither reject new exchanges nor be double-counted when swept. + uncounted bool +} + +// OIDCProvider implements OpenID Connect authentication with JTI replay protection. +// +// Replay Protection: +// The provider validates the 'jti' (JWT ID) claim to prevent token replay attacks. +// Each JTI is stored in memory until the token expires. If a token with a duplicate +// JTI is presented, it is rejected with ErrProviderTokenReplayed. +// +// The JTI cache is bounded by JTICacheMaxSize and uses automatic TTL-based expiration. +// A token without a usable jti claim is tracked by the token's own fingerprint +// instead, so a repeated presentation is still detected as a replay. +// +// Thread Safety: +// All methods are safe for concurrent use. The JTI cache uses atomic LoadOrStore +// operations to prevent TOCTOU race conditions. type OIDCProvider struct { name string config *OIDCConfig @@ -44,6 +83,32 @@ type OIDCProvider struct { jwksFetchedAt time.Time resolvedJWKSUri string discoveryFailed bool + + // JTI replay protection fields. jtiStore and jtiCount are allocated once + // in NewOIDCProvider and survive re-initialization, so a config refresh + // cannot reopen already-consumed tokens for replay. + jtiEnabled atomic.Bool + jtiStore *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys + jtiCount *atomic.Int64 // entries in jtiStore; shared with the store + jtiMaxSize atomic.Int64 + jtiLastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups + + // jtiStoreMu guards the jtiStore/jtiCount pair: replay-state handoffs + // (AdoptReplayStateFrom, MergeReplayStateFrom) repoint them under the + // write lock while the consume path holds a read lock, so a consume can + // never land in a store the handoff already drained and abandoned. + // It is always acquired before jtiSweepMu, never while holding it. + jtiStoreMu sync.RWMutex + + // jtiMu guards the lifecycle of the cleanup worker: jtiCleanupCancel and + // jtiCleanupDone. The worker itself only touches jtiStore/jtiCount, so + // holding jtiMu while waiting for it is safe. + jtiMu sync.Mutex + jtiCleanupCancel context.CancelFunc + jtiCleanupDone chan struct{} // Closed when cleanup goroutine exits + + // jtiSweepMu serializes cache sweeps so at most one scan runs at a time. + jtiSweepMu sync.Mutex } // OIDCConfig holds OIDC provider configuration @@ -107,6 +172,13 @@ type OIDCConfig struct { // TLSInsecureSkipVerify controls whether to skip TLS verification. // WARNING: Should only be used in development/testing environments. Never use in production. TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty"` + + // JTIReplayProtectionEnabled enables token replay protection via JTI tracking (default: true) + // Use pointer to distinguish between "not set" (nil = default true) and "explicitly false" + JTIReplayProtectionEnabled *bool `json:"jtiReplayProtectionEnabled,omitempty"` + + // JTICacheMaxSize sets maximum number of JTI entries to cache (default: 10000) + JTICacheMaxSize int `json:"jtiCacheMaxSize,omitempty"` } // PrincipalTagsClaim is the AWS-defined namespace claim that carries @@ -319,6 +391,8 @@ func NewOIDCProvider(name string) *OIDCProvider { return &OIDCProvider{ name: name, httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect}, + jtiStore: &sync.Map{}, + jtiCount: &atomic.Int64{}, } } @@ -429,10 +503,62 @@ func (p *OIDCProvider) Initialize(config interface{}) error { CheckRedirect: refuseDowngradeRedirect, } + // Configure JTI replay protection last, so a config failure above cannot + // leave a running worker behind. A previous worker is always stopped, + // whether the new configuration enables protection or disables it. + // The consumed-token store survives re-initialization: resetting it would + // let already-seen tokens be replayed against the refreshed provider. + jtiEnabled := true + if oidcConfig.JTIReplayProtectionEnabled != nil { + jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled + } + + p.jtiMu.Lock() + p.stopJTICleanupWorkerLocked() + if jtiEnabled { + maxSize := int64(oidcConfig.JTICacheMaxSize) + if maxSize <= 0 { + maxSize = defaultJTICacheSize + } + // the bound must land before the flag: a consume that sees enabled + // with a zero bound would reject the first token it registers + p.jtiMaxSize.Store(maxSize) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + p.jtiCleanupCancel = cancel + p.jtiCleanupDone = done + p.jtiEnabled.Store(true) + p.jtiMu.Unlock() + + go p.cleanupExpiredJTIs(ctx, done, defaultCleanupInterval) + glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", p.name, maxSize) + } else { + p.jtiEnabled.Store(false) + p.jtiMu.Unlock() + glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", p.name) + } + // For testing, we'll skip the actual OIDC client initialization return nil } +// stopJTICleanupWorkerLocked cancels the current cleanup goroutine and waits +// briefly for it to exit. Caller must hold jtiMu. +func (p *OIDCProvider) stopJTICleanupWorkerLocked() { + if p.jtiCleanupCancel == nil { + return + } + p.jtiCleanupCancel() + select { + case <-p.jtiCleanupDone: + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", p.name) + case <-time.After(2 * time.Second): + glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", p.name) + } + p.jtiCleanupCancel = nil + p.jtiCleanupDone = nil +} + // validateConfig validates the OIDC configuration func (p *OIDCProvider) validateConfig(config *OIDCConfig) error { if config.Issuer == "" { @@ -479,6 +605,18 @@ func (p *OIDCProvider) allowedAudiences() []string { // Authenticate authenticates a user with an OIDC token func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*providers.ExternalIdentity, error) { + return p.authenticate(ctx, token, false) +} + +// AuthenticateSingleUse authenticates and consumes the token so a second +// presentation is rejected as a replay. The web-identity exchange uses it, +// where a token buys a session exactly once; callers that validate the same +// token on every request use Authenticate. +func (p *OIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) { + return p.authenticate(ctx, token, true) +} + +func (p *OIDCProvider) authenticate(ctx context.Context, token string, singleUse bool) (*providers.ExternalIdentity, error) { if !p.initialized { return nil, fmt.Errorf("provider not initialized") } @@ -488,7 +626,7 @@ func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*provide } // Validate token and get claims - claims, err := p.ValidateToken(ctx, token) + claims, err := p.validateToken(ctx, token, singleUse) if err != nil { return nil, err } @@ -655,6 +793,16 @@ func (p *OIDCProvider) getUserInfoWithToken(ctx context.Context, userID, accessT // ValidateToken validates an OIDC JWT token func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) { + return p.validateToken(ctx, token, false) +} + +// ValidateTokenSingleUse validates and consumes the token: a second +// presentation of it is rejected as a replay. See AuthenticateSingleUse. +func (p *OIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) { + return p.validateToken(ctx, token, true) +} + +func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUse bool) (*providers.TokenClaims, error) { if !p.initialized { return nil, fmt.Errorf("provider not initialized") } @@ -787,6 +935,87 @@ func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*provid tokenClaims.Claims[key] = value } + // Extract and validate JTI claim for replay protection. The check runs + // only for single-use callers: a bearer token is meant to be presented on + // every request, so validating it must not consume it. + if singleUse && p.jtiEnabled.Load() { + jti, _ := claims["jti"].(string) + key := jti + if key == "" { + // Providers that issue no jti cannot be tracked by claim, so the + // signed input keys the entry: a repeated presentation of the same + // claims is a replay even when the signature bytes differ (e.g. + // ECDSA s/N-s malleability). + signingInput := token + if i := strings.LastIndexByte(token, '.'); i > 0 { + signingInput = token[:i] + } + sum := sha256.Sum256([]byte(signingInput)) + key = tokenFingerprintPrefix + hex.EncodeToString(sum[:]) + } + + expiresAt := tokenClaims.ExpiresAt + if expiresAt.IsZero() { + // a cache slot is freed only by expiry, so a token without exp + // would hold it forever and eventually exhaust the bound + return nil, fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims) + } + expiresAt = expiresAt.Add(clockSkewTolerance) + if !time.Now().Before(expiresAt) { + return nil, fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired) + } + + entry := &jtiEntry{ + subject: subject, + expiresAt: expiresAt, + } + + // The read lock pins the store/count pair for the whole test-and-set + // so a replay-state handoff cannot repoint them mid-check and strand + // this consume in an abandoned store. + p.jtiStoreMu.RLock() + store, count := p.jtiStore, p.jtiCount + + // Atomic test-and-set detects replays before any capacity work runs, + // so a replayed token cannot trigger cleanup scans. + if _, loaded := store.LoadOrStore(key, entry); loaded { + p.jtiStoreMu.RUnlock() + glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)", + p.name, jtiLogKey(key)) + return nil, fmt.Errorf("%w: jti %s has already been used", + providers.ErrProviderTokenReplayed, jtiLogKey(key)) + } + + // The count is incremented as part of the insert, so concurrent + // validations cannot all slip under the limit: whoever pushes the + // count over evicts their own entry and is rejected. + newCount := count.Add(1) + maxSize := p.jtiMaxSize.Load() + if newCount > maxSize { + // Over the bound: expired entries may still occupy slots, so + // reclaim them with a synchronized sweep before rejecting. + p.sweepJTIStore(store, count) + if count.Load() > maxSize { + if _, loaded := store.LoadAndDelete(key); loaded { + count.Add(-1) + } + p.jtiStoreMu.RUnlock() + glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+ + "This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.", + p.name, maxSize) + return nil, fmt.Errorf("JTI cache capacity exceeded") + } + } else if newCount >= maxSize*9/10 { + glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup", + p.name, (newCount*100)/maxSize, newCount, maxSize) + go p.performEagerCleanup() + } + p.jtiStoreMu.RUnlock() + + glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)", + p.name, jtiLogKey(key), expiresAt, newCount) + } + return tokenClaims, nil } @@ -1215,3 +1444,195 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p return identity } + +// evictExpiredJTIs removes entries whose expiry has passed and returns how +// many it deleted. Caller must hold jtiSweepMu. +func (p *OIDCProvider) evictExpiredJTIs(now time.Time, store *sync.Map, count *atomic.Int64) int { + removed := 0 + store.Range(func(key, value interface{}) bool { + expiresAt := value.(*jtiEntry).expiresAt + if !expiresAt.IsZero() && now.After(expiresAt) { + // LoadAndDelete reports whether this call actually removed the + // entry, so concurrent deletes cannot decrement the count twice. + if v, loaded := store.LoadAndDelete(key); loaded { + if !v.(*jtiEntry).uncounted { + count.Add(-1) + } + removed++ + } + } + return true + }) + return removed +} + +// replayState returns the provider's current store/count pair. +func (p *OIDCProvider) replayState() (*sync.Map, *atomic.Int64) { + p.jtiStoreMu.RLock() + defer p.jtiStoreMu.RUnlock() + return p.jtiStore, p.jtiCount +} + +// sweepJTIStore runs one serialized expiry scan over the given store. It only +// takes jtiSweepMu, so callers already holding jtiStoreMu may use it. +func (p *OIDCProvider) sweepJTIStore(store *sync.Map, count *atomic.Int64) { + p.jtiSweepMu.Lock() + defer p.jtiSweepMu.Unlock() + p.jtiLastSweep.Store(time.Now().UnixNano()) + if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { + glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)", + p.name, removed, count.Load()) + } +} + +// sweepExpiredJTIs runs a scan that is guaranteed to complete before the +// caller proceeds; the capacity-rejection path uses it so a fresh token is +// not refused while expired entries still hold slots. +func (p *OIDCProvider) sweepExpiredJTIs() { + store, count := p.replayState() + p.sweepJTIStore(store, count) +} + +// performEagerCleanup performs an opportunistic cleanup of expired JTI +// entries when the cache approaches capacity. At most one sweep runs at a +// time and no more often than eagerCleanupMinInterval, so a burst of +// requests near the limit cannot each turn into a full-cache scan. +func (p *OIDCProvider) performEagerCleanup() { + if !p.jtiEnabled.Load() { + return + } + if time.Since(time.Unix(0, p.jtiLastSweep.Load())) < eagerCleanupMinInterval { + return + } + store, count := p.replayState() + if !p.jtiSweepMu.TryLock() { + return + } + defer p.jtiSweepMu.Unlock() + p.jtiLastSweep.Store(time.Now().UnixNano()) + + beforeCount := count.Load() + if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { + glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)", + p.name, removed, beforeCount, count.Load()) + } else { + glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+ + "Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.", + p.name) + } +} + +// cleanupExpiredJTIs periodically removes expired JTI entries from the cache. +// This prevents memory growth by removing JTIs after their tokens have expired. +// Runs in a background goroutine until ctx is cancelled; done is closed on exit. +func (p *OIDCProvider) cleanupExpiredJTIs(ctx context.Context, done chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", p.name, interval) + + for { + select { + case <-ctx.Done(): + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", p.name) + return + case <-ticker.C: + p.sweepExpiredJTIs() + } + } +} + +// Shutdown gracefully stops the JTI cleanup goroutine. +// Should be called when the provider is being shut down to prevent goroutine leaks. +func (p *OIDCProvider) Shutdown(ctx context.Context) error { + p.jtiMu.Lock() + cancel, done := p.jtiCleanupCancel, p.jtiCleanupDone + p.jtiMu.Unlock() + + if cancel == nil { + return nil + } + cancel() + + select { + case <-done: + glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", p.name) + case <-ctx.Done(): + glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", p.name, ctx.Err()) + return ctx.Err() + } + + p.jtiMu.Lock() + if p.jtiCleanupDone == done { + p.jtiCleanupCancel = nil + p.jtiCleanupDone = nil + } + p.jtiMu.Unlock() + return nil +} + +// AdoptReplayStateFrom takes over the consumed-token store of the provider it +// replaces, so tokens it already saw stay rejected across the rebuild. The +// caller must shut down the replaced provider; only one cleanup worker may +// sweep the shared store at a time. Call before the new provider serves +// validators; jtiSweepMu excludes its own cleanup worker during the swap. +func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) { + if old == nil || old == p { + return + } + p.jtiStoreMu.Lock() + p.jtiSweepMu.Lock() + p.jtiStore, p.jtiCount = old.jtiStore, old.jtiCount + p.jtiSweepMu.Unlock() + p.jtiStoreMu.Unlock() +} + +// MergeReplayStateFrom hands this provider's store to another provider and +// then folds the other's consumed-token entries in, so rejections the other +// recorded keep holding when its records move to a shared provider. Repointing +// first means tokens the other still consumes — in-flight exchanges or ones +// routed to it before the provider map updates — land in the shared store +// instead of a store this merge already drained. Unlike AdoptReplayStateFrom +// it swaps nothing of its own, so it is safe on a provider that is already +// serving validators. +func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) { + if other == nil || other == p { + return + } + p.jtiStoreMu.RLock() + defer p.jtiStoreMu.RUnlock() + store, count := p.jtiStore, p.jtiCount + + other.jtiStoreMu.Lock() + oldStore := other.jtiStore + if oldStore == store { + other.jtiStoreMu.Unlock() + return + } + other.jtiStore, other.jtiCount = store, count + other.jtiStoreMu.Unlock() + + p.jtiSweepMu.Lock() + defer p.jtiSweepMu.Unlock() + now := time.Now() + oldStore.Range(func(key, value interface{}) bool { + entry := value.(*jtiEntry) + if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) { + return true + } + // A consumed token is never dropped to fit the bound: merged entries + // stay but do not charge the count, so a store filled past the bound + // by the merge cannot reject new exchanges while it drains by expiry. + store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true}) + return true + }) +} + +// jtiLogKey redacts a token identifier for logs and error strings: a stable +// fingerprint that still lets operators correlate repeated presentations. +func jtiLogKey(jti string) string { + sum := sha256.Sum256([]byte(jti)) + return hex.EncodeToString(sum[:8]) +} diff --git a/weed/iam/oidc/oidc_provider_test.go b/weed/iam/oidc/oidc_provider_test.go index b226929d2f..3744ec3138 100644 --- a/weed/iam/oidc/oidc_provider_test.go +++ b/weed/iam/oidc/oidc_provider_test.go @@ -8,9 +8,14 @@ import ( "crypto/rsa" "encoding/base64" "encoding/json" + "errors" + "fmt" + "math/big" "net/http" "net/http/httptest" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -547,42 +552,42 @@ func TestOIDCProviderUserInfo(t *testing.T) { // Helper functions for testing -func generateTestKeys(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey) { +func generateTestKeys(tb testing.TB) (*rsa.PrivateKey, *rsa.PublicKey) { privateKey, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) + require.NoError(tb, err) return privateKey, &privateKey.PublicKey } -func generateTestECKeys(t *testing.T) (*ecdsa.PrivateKey, *ecdsa.PublicKey) { +func generateTestECKeys(tb testing.TB) (*ecdsa.PrivateKey, *ecdsa.PublicKey) { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) + require.NoError(tb, err) return privateKey, &privateKey.PublicKey } -func createTestJWT(t *testing.T, privateKey *rsa.PrivateKey, claims jwt.MapClaims) string { +func createTestJWT(tb testing.TB, privateKey *rsa.PrivateKey, claims jwt.MapClaims) string { token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) token.Header["kid"] = "test-key-id" tokenString, err := token.SignedString(privateKey) - require.NoError(t, err) + require.NoError(tb, err) return tokenString } -func createTestECDSAJWT(t *testing.T, privateKey *ecdsa.PrivateKey, claims jwt.MapClaims) string { +func createTestECDSAJWT(tb testing.TB, privateKey *ecdsa.PrivateKey, claims jwt.MapClaims) string { token := jwt.NewWithClaims(jwt.SigningMethodES256, claims) token.Header["kid"] = "test-ec-key-id" tokenString, err := token.SignedString(privateKey) - require.NoError(t, err) + require.NoError(tb, err) return tokenString } -func encodePublicKey(t *testing.T, publicKey *rsa.PublicKey) string { +func encodePublicKey(tb testing.TB, publicKey *rsa.PublicKey) string { // Properly encode the RSA modulus (N) as base64url return base64.RawURLEncoding.EncodeToString(publicKey.N.Bytes()) } -func encodeECPublicKey(t *testing.T, publicKey *ecdsa.PublicKey) (string, string) { +func encodeECPublicKey(tb testing.TB, publicKey *ecdsa.PublicKey) (string, string) { // RFC 7518 §6.2.1.2 requires EC coordinates to be zero-padded to the full field size curveParams := publicKey.Curve.Params() size := (curveParams.BitSize + 7) / 8 @@ -597,7 +602,7 @@ func encodeECPublicKey(t *testing.T, publicKey *ecdsa.PublicKey) (string, string base64.RawURLEncoding.EncodeToString(yPadded) } -func setupOIDCTestServer(t *testing.T, publicKey *rsa.PublicKey) *httptest.Server { +func setupOIDCTestServer(tb testing.TB, publicKey *rsa.PublicKey) *httptest.Server { jwks := map[string]interface{}{ "keys": []map[string]interface{}{ { @@ -605,7 +610,7 @@ func setupOIDCTestServer(t *testing.T, publicKey *rsa.PublicKey) *httptest.Serve "kid": "test-key-id", "use": "sig", "alg": "RS256", - "n": encodePublicKey(t, publicKey), + "n": encodePublicKey(tb, publicKey), "e": "AQAB", }, }, @@ -660,3 +665,1066 @@ func setupOIDCTestServer(t *testing.T, publicKey *rsa.PublicKey) *httptest.Serve } })) } + +func TestOIDCProviderJTIReplayProtection(t *testing.T) { + // Setup mock OIDC provider with JTI protection enabled + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 100, + } + + err := provider.Initialize(config) + require.NoError(t, err) + + // Test 1: First use of token with JTI should succeed + jti := "unique-token-id-12345" + token1 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": jti, + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + claims1, err := provider.ValidateTokenSingleUse(context.Background(), token1) + require.NoError(t, err) + assert.Equal(t, "user123", claims1.Subject) + assert.Equal(t, jti, claims1.Claims["jti"]) + + // Test 2: Replay of same token should be rejected + claims2, err := provider.ValidateTokenSingleUse(context.Background(), token1) + assert.Error(t, err) + assert.Nil(t, claims2) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed)) + assert.NotContains(t, err.Error(), jti, "error should not expose the raw token identifier") + + // Test 3: Different token with different JTI should succeed + jti2 := "different-token-id-67890" + token2 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user456", + "jti": jti2, + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + claims3, err := provider.ValidateTokenSingleUse(context.Background(), token2) + require.NoError(t, err) + assert.Equal(t, "user456", claims3.Subject) + + // Test 4: Token without JTI is tracked by the token's own fingerprint, + // so presenting the same token again is still a replay + tokenNoJTI := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user789", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + claims4, err := provider.ValidateTokenSingleUse(context.Background(), tokenNoJTI) + require.NoError(t, err) + assert.Equal(t, "user789", claims4.Subject) + + _, err = provider.ValidateTokenSingleUse(context.Background(), tokenNoJTI) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "a jti-less token must not stay reusable while protection is enabled") + + // Test 5: Token with a JTI but no exp is rejected — a cache slot frees + // only by expiry, so an expiry-less entry would hold it forever and a + // burst of distinct tokens could exhaust the bound permanently. + tokenNoExp := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "userNoExp", + "jti": "no-exp-jti", + "iat": time.Now().Unix(), + }) + + claims5, err := provider.ValidateTokenSingleUse(context.Background(), tokenNoExp) + assert.Nil(t, claims5) + assert.True(t, errors.Is(err, providers.ErrProviderMissingClaims)) +} + +func TestOIDCProviderJTIFallbackResistsSignatureMalleability(t *testing.T) { + privateKey, publicKey := generateTestECKeys(t) + x, y := encodeECPublicKey(t, publicKey) + + jwks := map[string]interface{}{ + "keys": []map[string]interface{}{ + { + "kty": "EC", + "kid": "test-ec-key-id", + "use": "sig", + "alg": "ES256", + "crv": "P-256", + "x": x, + "y": y, + }, + }, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/openid_configuration" { + json.NewEncoder(w).Encode(map[string]interface{}{ + "issuer": "http://" + r.Host, + "jwks_uri": "http://" + r.Host + "/jwks", + }) + } else if r.URL.Path == "/jwks" { + json.NewEncoder(w).Encode(jwks) + } + })) + defer server.Close() + + provider := NewOIDCProvider("test-oidc-ecdsa-malleable") + err := provider.Initialize(&OIDCConfig{ + Issuer: server.URL, + ClientID: "test-client", + JWKSUri: server.URL + "/jwks", + }) + require.NoError(t, err) + + token := createTestECDSAJWT(t, privateKey, jwt.MapClaims{ + "iss": server.URL, + "aud": "test-client", + "sub": "user-malleable", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + _, err = provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + + // The same signed claims under a malleated ECDSA signature (s -> N-s) + // must still be a replay, not a fresh exchange. + malleated := malleateECDSAJWTSignature(t, token) + require.NotEqual(t, token, malleated) + + _, err = provider.ValidateTokenSingleUse(context.Background(), malleated) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "a signature-malleated jti-less token must not bypass replay detection") +} + +func malleateECDSAJWTSignature(tb testing.TB, tokenString string) string { + parts := strings.Split(tokenString, ".") + require.Len(tb, parts, 3) + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + require.NoError(tb, err) + half := len(sig) / 2 + r := new(big.Int).SetBytes(sig[:half]) + s := new(big.Int).SetBytes(sig[half:]) + s.Sub(elliptic.P256().Params().N, s) + malleated := make([]byte, len(sig)) + r.FillBytes(malleated[:half]) + s.FillBytes(malleated[half:]) + parts[2] = base64.RawURLEncoding.EncodeToString(malleated) + return strings.Join(parts, ".") +} + +func TestOIDCProviderJTIReplayProtectionDisabled(t *testing.T) { + // Setup provider with JTI protection DISABLED + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-no-jti") + jtiDisabled := false + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiDisabled, // Explicitly disabled + } + + err := provider.Initialize(config) + require.NoError(t, err) + + // Token with JTI can be used multiple times when protection is disabled + jti := "reusable-token-id" + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": jti, + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + // First use - should succeed + claims1, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + assert.Equal(t, "user123", claims1.Subject) + + // Second use - should ALSO succeed (no replay protection) + claims2, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + assert.Equal(t, "user123", claims2.Subject) +} + +func TestOIDCProviderJTIExpiration(t *testing.T) { + // Test that JTI cache entries expire with token + // This test also verifies default behavior (JTIReplayProtectionEnabled not set = enabled by default) + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc") + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + // JTIReplayProtectionEnabled not set - should default to true (secure by default) + } + + err := provider.Initialize(config) + require.NoError(t, err) + + // Create token that expires in 2 seconds + jti := "short-lived-token" + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": jti, + "exp": time.Now().Add(2 * time.Second).Unix(), + "iat": time.Now().Unix(), + }) + + // First use - should succeed + claims1, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + assert.Equal(t, "user123", claims1.Subject) + + // Immediate replay - should be rejected + _, err = provider.ValidateTokenSingleUse(context.Background(), token) + assert.Error(t, err) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed)) + + // Wait for token AND cache entry to expire + time.Sleep(3 * time.Second) + + // Now validation should fail due to EXPIRATION (not replay) + _, err = provider.ValidateTokenSingleUse(context.Background(), token) + assert.Error(t, err) + assert.True(t, errors.Is(err, providers.ErrProviderTokenExpired)) +} + +func TestOIDCProviderJTIReplayProtectionConcurrent(t *testing.T) { + // Test concurrent validation to ensure no TOCTOU race conditions + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-concurrent") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 1000, + } + require.NoError(t, provider.Initialize(config)) + + jti := "concurrent-test-jti" + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": jti, + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + // Launch 100 concurrent validations with the same token + const numGoroutines = 100 + var successCount, replayCount, otherErrorCount atomic.Int32 + + var wg sync.WaitGroup + wg.Add(numGoroutines) + + // All goroutines start at roughly the same time + startBarrier := make(chan struct{}) + + for i := 0; i < numGoroutines; i++ { + go func() { + defer wg.Done() + <-startBarrier // Wait for start signal + + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + if err == nil { + successCount.Add(1) + } else if errors.Is(err, providers.ErrProviderTokenReplayed) { + replayCount.Add(1) + } else { + otherErrorCount.Add(1) + t.Logf("Unexpected error: %v", err) + } + }() + } + + close(startBarrier) // Start all goroutines + wg.Wait() + + // Exactly one should succeed, all others should be replay errors + assert.Equal(t, int32(1), successCount.Load(), + "Exactly one validation should succeed (atomic test-and-set)") + assert.Equal(t, int32(numGoroutines-1), replayCount.Load(), + "All other validations should be replay errors") + assert.Equal(t, int32(0), otherErrorCount.Load(), + "No other errors should occur") +} + +func TestOIDCProviderJTICacheMaxSize(t *testing.T) { + // Test that cache respects max size limit + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-maxsize") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 10, // Small cache for testing + } + require.NoError(t, provider.Initialize(config)) + + // Add tokens up to the limit + for i := 0; i < 10; i++ { + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": fmt.Sprintf("jti-%d", i), + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err, "Should accept token %d", i) + } + + // Next token should be rejected due to cache full + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "jti-overflow", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cache capacity exceeded") +} + +func TestOIDCProviderJTICleanup(t *testing.T) { + // Test that cleanup goroutine logic correctly identifies expired entries + // NOTE: JWT library allows clock skew, so tokens can't be truly expired + // We test the cleanup logic without actually waiting for expiration + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-cleanup") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + } + require.NoError(t, provider.Initialize(config)) + defer provider.Shutdown(context.Background()) + + // Create token that expires in 1 hour + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "cleanup-test-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + // Validation should succeed + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + + // Check cache size is 1 + assert.Equal(t, int64(1), provider.jtiCount.Load()) + + // Verify entry exists and check its expiration includes clock skew + val, ok := provider.jtiStore.Load("cleanup-test-jti") + require.True(t, ok, "JTI entry should exist") + entry := val.(*jtiEntry) + + // Entry should expire around exp + clock skew tolerance (1 hour + 5 minutes) + expectedExpiry := time.Now().Add(time.Hour + 5*time.Minute) + assert.WithinDuration(t, expectedExpiry, entry.expiresAt, 10*time.Second, + "Entry expiry should be token exp + clock skew tolerance") + + // Verify entry is not yet expired (cleanup shouldn't remove it) + now := time.Now() + assert.True(t, now.Before(entry.expiresAt), + "Entry should not be expired yet") + + // Run the periodic cleanup with a short interval and an expired entry: + // both the entry and the count must actually be cleared + cleanupCtx, cleanupCancel := context.WithCancel(context.Background()) + cleanupDone := make(chan struct{}) + go provider.cleanupExpiredJTIs(cleanupCtx, cleanupDone, 20*time.Millisecond) + defer func() { + cleanupCancel() + <-cleanupDone + }() + + provider.jtiStore.Store("expired-cleanup-jti", &jtiEntry{ + subject: "expired-user", + expiresAt: time.Now().Add(-time.Minute), + }) + provider.jtiCount.Add(1) + + require.Eventually(t, func() bool { + _, stillThere := provider.jtiStore.Load("expired-cleanup-jti") + return !stillThere && provider.jtiCount.Load() == 1 // only the valid entry remains + }, 2*time.Second, 10*time.Millisecond, "periodic cleanup should evict the expired entry") +} + +func TestOIDCProviderReinitialization(t *testing.T) { + // Test that re-initialization properly cancels the old cleanup goroutine + // to prevent goroutine leaks + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-reinit") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + } + + // First initialization + require.NoError(t, provider.Initialize(config)) + require.NotNil(t, provider.jtiCleanupCancel, "First cleanup cancel should be set") + + // Create and validate a token to ensure JTI protection is working + token1 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user1", + "jti": "token-1", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := provider.ValidateTokenSingleUse(context.Background(), token1) + require.NoError(t, err, "First token should validate successfully") + + // Second initialization should not cause errors + require.NoError(t, provider.Initialize(config)) + require.NotNil(t, provider.jtiCleanupCancel, "Second cleanup cancel should be set") + + // The JTI cache survives re-initialization, so the consumed token is + // still detected as a replay rather than being usable again + _, err = provider.ValidateTokenSingleUse(context.Background(), token1) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "Re-initialization must not reopen a consumed token for replay") + + // Third initialization to ensure it works multiple times + require.NoError(t, provider.Initialize(config)) + require.NotNil(t, provider.jtiCleanupCancel, "Third cleanup cancel should be set") + + // Verify JTI replay protection still works after re-initialization + token2 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user2", + "jti": "token-2", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err = provider.ValidateTokenSingleUse(context.Background(), token2) + require.NoError(t, err, "New token should validate") + + // Replay should still be detected + _, err = provider.ValidateTokenSingleUse(context.Background(), token2) + assert.Error(t, err, "Replay should be detected") + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), "Should be a replay error") + + // Final cleanup + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err = provider.Shutdown(ctx) + assert.NoError(t, err) +} + +func TestOIDCProviderJTIEagerCleanup(t *testing.T) { + // Test the eager cleanup mechanism when cache approaches capacity + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-eager-cleanup") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 10, // Small cache for testing + } + require.NoError(t, provider.Initialize(config)) + defer provider.Shutdown(context.Background()) + + // Manually add expired entries to the cache + for i := 0; i < 5; i++ { + jti := fmt.Sprintf("expired-jti-%d", i) + entry := &jtiEntry{ + subject: fmt.Sprintf("expired-user-%d", i), + expiresAt: time.Now().Add(-time.Hour), // Already expired + } + provider.jtiStore.Store(jti, entry) + provider.jtiCount.Add(1) + } + + assert.Equal(t, int64(5), provider.jtiCount.Load(), "Should have 5 expired entries") + + // Perform eager cleanup - should remove all expired entries + provider.performEagerCleanup() + + assert.Equal(t, int64(0), provider.jtiCount.Load(), + "All expired entries should be removed by eager cleanup") + + // Now test that adding valid tokens works after cleanup + for i := 0; i < 9; i++ { + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": fmt.Sprintf("user-%d", i), + "jti": fmt.Sprintf("valid-jti-%d", i), + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + } + + // Should have 9 valid entries now (90% of capacity, triggers eager cleanup warning) + assert.Equal(t, int64(9), provider.jtiCount.Load(), "Should have 9 valid entries") + + // 10th token should succeed (at capacity but not over) + token10 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user-10", + "jti": "valid-jti-10", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := provider.ValidateTokenSingleUse(context.Background(), token10) + require.NoError(t, err, "10th token should validate (at capacity)") + + // 11th token should fail (over capacity) + token11 := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user-11", + "jti": "valid-jti-11", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err = provider.ValidateTokenSingleUse(context.Background(), token11) + assert.Error(t, err, "11th token should fail (over capacity)") + assert.Contains(t, err.Error(), "capacity exceeded", "Should indicate capacity exceeded") +} + +func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { + // Test that concurrent cleanups don't cause jtiCount to go negative + // due to double-decrement of the same entry + _, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-cleanup-race") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 100, + } + require.NoError(t, provider.Initialize(config)) + defer provider.Shutdown(context.Background()) + + // Manually add expired entries to the cache + for i := 0; i < 50; i++ { + jti := fmt.Sprintf("expired-jti-%d", i) + entry := &jtiEntry{ + subject: fmt.Sprintf("user-%d", i), + expiresAt: time.Now().Add(-time.Hour), // Already expired + } + provider.jtiStore.Store(jti, entry) + provider.jtiCount.Add(1) + } + + initialCount := provider.jtiCount.Load() + assert.Equal(t, int64(50), initialCount, "Should have 50 expired entries") + + // Run multiple concurrent cleanups + var wg sync.WaitGroup + numCleanups := 10 + + for i := 0; i < numCleanups; i++ { + wg.Add(1) + go func() { + defer wg.Done() + provider.performEagerCleanup() + }() + } + + wg.Wait() + + // Counter should be 0 (all entries removed), never negative + finalCount := provider.jtiCount.Load() + assert.Equal(t, int64(0), finalCount, + "Counter should be 0 after concurrent cleanups, got %d", finalCount) + assert.GreaterOrEqual(t, finalCount, int64(0), + "Counter should never go negative, got %d", finalCount) + + // Verify store is actually empty + storeCount := 0 + provider.jtiStore.Range(func(key, value interface{}) bool { + storeCount++ + return true + }) + assert.Equal(t, 0, storeCount, "Store should be empty") +} + +func TestOIDCProviderShutdown(t *testing.T) { + // Test graceful shutdown + _, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-shutdown") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + } + require.NoError(t, provider.Initialize(config)) + + // Shutdown should complete without error + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Measure shutdown time to ensure it doesn't block unnecessarily + start := time.Now() + err := provider.Shutdown(ctx) + duration := time.Since(start) + + assert.NoError(t, err) + // Shutdown should be quick (goroutine exits immediately on cancel) + // Allow 1 second for goroutine scheduling overhead + assert.Less(t, duration, 1*time.Second, + "Shutdown should not block for fixed timeout, took %v", duration) + + // Second shutdown should be idempotent + err = provider.Shutdown(ctx) + assert.NoError(t, err) +} + +func TestOIDCProviderShutdownTimeout(t *testing.T) { + // Test that shutdown respects caller's context timeout + _, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-shutdown-timeout") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + } + require.NoError(t, provider.Initialize(config)) + + // Use a very short timeout to test timeout handling + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + // Wait a bit to ensure context expires + time.Sleep(10 * time.Millisecond) + + err := provider.Shutdown(ctx) + // Should return context error if timeout occurs + if err != nil { + assert.ErrorIs(t, err, context.DeadlineExceeded, + "Should return deadline exceeded error when context times out") + } + // Note: goroutine may exit before timeout, so error is optional +} + +// Benchmark tests + +func BenchmarkJTIValidation(b *testing.B) { + // Benchmark JTI validation performance with unique tokens + privateKey, publicKey := generateTestKeys(b) + jwksServer := setupOIDCTestServer(b, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("bench-oidc") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: b.N + 1024, + } + require.NoError(b, provider.Initialize(config)) + defer provider.Shutdown(context.Background()) + + // Pre-generate unique tokens + tokens := make([]string, b.N) + for i := 0; i < b.N; i++ { + tokens[i] = createTestJWT(b, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": fmt.Sprintf("jti-%d", i), + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + } + + // Parallel workers draw distinct token indices so each iteration + // validates a new token rather than a replay or capacity rejection. + var tokenIndex atomic.Int64 + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i := int(tokenIndex.Add(1)-1) % len(tokens) + if _, err := provider.ValidateTokenSingleUse(context.Background(), tokens[i]); err != nil { + b.Errorf("unexpected validation error: %v", err) + } + } + }) +} + +func BenchmarkJTIValidationWithReplay(b *testing.B) { + // Benchmark replay detection performance + privateKey, publicKey := generateTestKeys(b) + jwksServer := setupOIDCTestServer(b, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("bench-oidc-replay") + jtiEnabled := true + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + JTICacheMaxSize: 100000, + } + require.NoError(b, provider.Initialize(config)) + defer provider.Shutdown(context.Background()) + + // Create one token that will be replayed + token := createTestJWT(b, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "replay-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + // First validation to store the JTI + _, err := provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(b, err) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // All subsequent validations should be fast replay rejections + provider.ValidateTokenSingleUse(context.Background(), token) + } + }) +} + +func BenchmarkJTIValidationNoProtection(b *testing.B) { + // Benchmark validation without JTI protection (baseline) + privateKey, publicKey := generateTestKeys(b) + jwksServer := setupOIDCTestServer(b, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("bench-oidc-no-jti") + jtiDisabled := false + config := &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiDisabled, + } + require.NoError(b, provider.Initialize(config)) + + token := createTestJWT(b, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "no-protection-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + provider.ValidateTokenSingleUse(context.Background(), token) + } + }) +} + +func TestOIDCProviderBearerValidationDoesNotConsume(t *testing.T) { + // Bearer authentication validates the same token on every request, so it + // must not consume it; only the single-use exchange rejects a repeat. + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + provider := NewOIDCProvider("test-oidc-bearer") + jtiEnabled := true + require.NoError(t, provider.Initialize(&OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTIReplayProtectionEnabled: &jtiEnabled, + })) + defer provider.Shutdown(context.Background()) + + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "bearer-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + + for i := 0; i < 3; i++ { + _, err := provider.ValidateToken(context.Background(), token) + require.NoError(t, err, "bearer validation must not consume the token") + } + + _, err := provider.Authenticate(context.Background(), token) + require.NoError(t, err, "bearer authentication must not consume the token") + + // The same token then exchanged single-use is tracked and a repeat rejected. + _, err = provider.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + _, err = provider.ValidateTokenSingleUse(context.Background(), token) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed)) + + // Even after the exchange consumed it, bearer validation still accepts: + // the token itself remains valid until it expires. + _, err = provider.ValidateToken(context.Background(), token) + require.NoError(t, err) +} + +func TestAdoptReplayStateFromKeepsConsumedTokensRejected(t *testing.T) { + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + newConfig := func() *OIDCConfig { + return &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + } + } + + old := NewOIDCProvider("test-oidc-old") + require.NoError(t, old.Initialize(newConfig())) + + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "consumed-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := old.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + + rebuilt := NewOIDCProvider("test-oidc-new") + require.NoError(t, rebuilt.Initialize(newConfig())) + defer rebuilt.Shutdown(context.Background()) + + rebuilt.AdoptReplayStateFrom(old) + require.NoError(t, old.Shutdown(context.Background())) + + _, err = rebuilt.ValidateTokenSingleUse(context.Background(), token) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "a token consumed before the rebuild must stay rejected") +} + +// A token consumed by one provider must stay rejected by another provider for +// the same issuer once the consumed entries merge over: pooling a record onto +// a shared provider cannot reopen the replay window. +func TestMergeReplayStateFromKeepsConsumedTokensRejected(t *testing.T) { + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + newConfig := func() *OIDCConfig { + return &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + } + } + + first := NewOIDCProvider("test-oidc-first") + require.NoError(t, first.Initialize(newConfig())) + defer first.Shutdown(context.Background()) + + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "consumed-elsewhere-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := first.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err) + + second := NewOIDCProvider("test-oidc-second") + require.NoError(t, second.Initialize(newConfig())) + defer second.Shutdown(context.Background()) + + second.MergeReplayStateFrom(first) + _, err = second.ValidateTokenSingleUse(context.Background(), token) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "a token the superseded provider consumed must stay rejected after the merge") +} + +// The superseded provider may keep serving exchanges while the provider map +// updates; tokens it consumes after the handoff must land in the shared store +// or they would reopen for replay. +func TestMergeReplayStateFromSharesLiveStore(t *testing.T) { + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + newConfig := func() *OIDCConfig { + return &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + } + } + + first := NewOIDCProvider("test-oidc-first") + require.NoError(t, first.Initialize(newConfig())) + defer first.Shutdown(context.Background()) + + second := NewOIDCProvider("test-oidc-second") + require.NoError(t, second.Initialize(newConfig())) + defer second.Shutdown(context.Background()) + + second.MergeReplayStateFrom(first) + + token := createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": "late-consume-jti", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + _, err := first.ValidateTokenSingleUse(context.Background(), token) + require.NoError(t, err, "the superseded provider still accepts a fresh token") + + _, err = second.ValidateTokenSingleUse(context.Background(), token) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "a token consumed through the superseded provider after the handoff must stay rejected") +} + +// Merging two stores that are each near the bound must not reject new +// exchanges: consumed tokens are carried over without charging the bound, +// which still applies to tokens consumed through the live path. +func TestMergeReplayStateFromOverflowStillAdmits(t *testing.T) { + privateKey, publicKey := generateTestKeys(t) + jwksServer := setupOIDCTestServer(t, publicKey) + defer jwksServer.Close() + + newConfig := func() *OIDCConfig { + return &OIDCConfig{ + Issuer: jwksServer.URL, + ClientID: "test-client", + JWKSUri: jwksServer.URL + "/jwks", + JTICacheMaxSize: 4, + } + } + newToken := func(jti string) string { + return createTestJWT(t, privateKey, jwt.MapClaims{ + "iss": jwksServer.URL, + "aud": "test-client", + "sub": "user123", + "jti": jti, + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + }) + } + + first := NewOIDCProvider("test-oidc-first") + require.NoError(t, first.Initialize(newConfig())) + defer first.Shutdown(context.Background()) + for _, jti := range []string{"first-a", "first-b", "first-c"} { + _, err := first.ValidateTokenSingleUse(context.Background(), newToken(jti)) + require.NoError(t, err) + } + + second := NewOIDCProvider("test-oidc-second") + require.NoError(t, second.Initialize(newConfig())) + defer second.Shutdown(context.Background()) + for _, jti := range []string{"second-a", "second-b", "second-c"} { + _, err := second.ValidateTokenSingleUse(context.Background(), newToken(jti)) + require.NoError(t, err) + } + + second.MergeReplayStateFrom(first) + + _, err := second.ValidateTokenSingleUse(context.Background(), newToken("first-a")) + assert.True(t, errors.Is(err, providers.ErrProviderTokenReplayed), + "merged consumed tokens must stay rejected") + + _, err = second.ValidateTokenSingleUse(context.Background(), newToken("fresh-jti")) + assert.NoError(t, err, "merged excess must not reject a new exchange") +} diff --git a/weed/iam/providers/errors.go b/weed/iam/providers/errors.go index eeac47c521..02ae7df532 100644 --- a/weed/iam/providers/errors.go +++ b/weed/iam/providers/errors.go @@ -19,4 +19,7 @@ var ( // ErrProviderMissingClaims indicates that required claims are missing from the token ErrProviderMissingClaims = errors.New("provider: missing required claims") + + // ErrProviderTokenReplayed is returned when a token with duplicate JTI is detected + ErrProviderTokenReplayed = errors.New("provider: token has been replayed") ) diff --git a/weed/iam/sts/constants.go b/weed/iam/sts/constants.go index 9a4876069e..3e6e24df39 100644 --- a/weed/iam/sts/constants.go +++ b/weed/iam/sts/constants.go @@ -42,21 +42,23 @@ const ( // Configuration Field Names const ( - ConfigFieldFilerAddress = "filerAddress" - ConfigFieldBasePath = "basePath" - ConfigFieldIssuer = "issuer" - ConfigFieldClientID = "clientId" - ConfigFieldClientIDs = "clientIds" - ConfigFieldThumbprints = "thumbprints" - ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys" - ConfigFieldPolicyClaim = "policyClaim" - ConfigFieldClientSecret = "clientSecret" - ConfigFieldJWKSUri = "jwksUri" - ConfigFieldScopes = "scopes" - ConfigFieldUserInfoUri = "userInfoUri" - ConfigFieldRedirectUri = "redirectUri" - ConfigFieldTLSCACert = "tlsCaCert" - ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify" + ConfigFieldFilerAddress = "filerAddress" + ConfigFieldBasePath = "basePath" + ConfigFieldIssuer = "issuer" + ConfigFieldClientID = "clientId" + ConfigFieldClientIDs = "clientIds" + ConfigFieldThumbprints = "thumbprints" + ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys" + ConfigFieldPolicyClaim = "policyClaim" + ConfigFieldClientSecret = "clientSecret" + ConfigFieldJWKSUri = "jwksUri" + ConfigFieldScopes = "scopes" + ConfigFieldUserInfoUri = "userInfoUri" + ConfigFieldRedirectUri = "redirectUri" + ConfigFieldTLSCACert = "tlsCaCert" + ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify" + ConfigFieldJTIReplayProtectionEnabled = "jtiReplayProtectionEnabled" + ConfigFieldJTICacheMaxSize = "jtiCacheMaxSize" ) // Error Messages diff --git a/weed/iam/sts/oidc_provider_lookup_test.go b/weed/iam/sts/oidc_provider_lookup_test.go index 0281e65b33..aba144be2b 100644 --- a/weed/iam/sts/oidc_provider_lookup_test.go +++ b/weed/iam/sts/oidc_provider_lookup_test.go @@ -11,8 +11,8 @@ import ( // lookupOIDCProviderForAccount; the lookup never calls Authenticate. type stubIdentityProvider struct{ name string } -func (s *stubIdentityProvider) Name() string { return s.name } -func (s *stubIdentityProvider) Initialize(interface{}) error { return nil } +func (s *stubIdentityProvider) Name() string { return s.name } +func (s *stubIdentityProvider) Initialize(interface{}) error { return nil } func (s *stubIdentityProvider) Authenticate(context.Context, string) (*providers.ExternalIdentity, error) { return nil, nil } @@ -103,3 +103,51 @@ func TestLookupOIDCProviderForAccountFallsBackToStatic(t *testing.T) { t.Fatalf("expected static-config provider as last resort, got %v ok=%v", got, ok) } } + +// shutdownTrackingProvider records Shutdown calls so tests can observe which +// replaced providers the swap released. +type shutdownTrackingProvider struct { + stubIdentityProvider + shutdownCalls int32 +} + +func (s *shutdownTrackingProvider) Shutdown(context.Context) error { + s.shutdownCalls++ + return nil +} + +// A provider dropped from the managed map owns a background JTI cleanup +// worker; replacing the map must shut it down, while providers that stay in +// the map keep running. +func TestSetIAMManagedOIDCProvidersShutsDownDroppedProviders(t *testing.T) { + dropped := &shutdownTrackingProvider{stubIdentityProvider{name: "dropped"}, 0} + kept := &shutdownTrackingProvider{stubIdentityProvider{name: "kept"}, 0} + added := &shutdownTrackingProvider{stubIdentityProvider{name: "added"}, 0} + + s := &STSService{issuerToProvider: map[string]providers.IdentityProvider{}} + s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{ + "https://a.example.com": {{Provider: dropped}}, + "https://b.example.com": {{Provider: kept}, {AccountID: "111111111111", Provider: kept}}, + }) + + s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{ + "https://b.example.com": {{Provider: kept}}, + "https://c.example.com": {{Provider: added}}, + }) + + if dropped.shutdownCalls != 1 { + t.Fatalf("dropped provider was shut down %d times, want 1", dropped.shutdownCalls) + } + if kept.shutdownCalls != 0 { + t.Fatalf("kept provider was shut down %d times, want 0", kept.shutdownCalls) + } + if added.shutdownCalls != 0 { + t.Fatalf("newly added provider was shut down %d times, want 0", added.shutdownCalls) + } + + // Clearing the map releases the remaining providers. + s.SetIAMManagedOIDCProviders(nil) + if kept.shutdownCalls != 1 || added.shutdownCalls != 1 { + t.Fatalf("clearing the map shut down kept=%d added=%d, want 1 each", kept.shutdownCalls, added.shutdownCalls) + } +} diff --git a/weed/iam/sts/provider_factory.go b/weed/iam/sts/provider_factory.go index 63e27e8124..b68894eb58 100644 --- a/weed/iam/sts/provider_factory.go +++ b/weed/iam/sts/provider_factory.go @@ -1,6 +1,7 @@ package sts import ( + "encoding/json" "fmt" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -154,6 +155,21 @@ func (f *ProviderFactory) convertToOIDCConfig(configMap map[string]interface{}) config.TLSInsecureSkipVerify = tlsInsecureSkipVerify } + if enabled, ok := configMap[ConfigFieldJTIReplayProtectionEnabled].(bool); ok { + config.JTIReplayProtectionEnabled = &enabled + } + + switch maxSize := configMap[ConfigFieldJTICacheMaxSize].(type) { + case float64: + config.JTICacheMaxSize = int(maxSize) + case int: + config.JTICacheMaxSize = maxSize + case json.Number: + if n, err := maxSize.Int64(); err == nil { + config.JTICacheMaxSize = int(n) + } + } + // Convert claims mapping if claimsMapInterface, ok := configMap["claimsMapping"]; ok { claimsMap, err := f.convertToStringMap(claimsMapInterface) diff --git a/weed/iam/sts/provider_factory_jti_test.go b/weed/iam/sts/provider_factory_jti_test.go new file mode 100644 index 0000000000..93800113ff --- /dev/null +++ b/weed/iam/sts/provider_factory_jti_test.go @@ -0,0 +1,31 @@ +package sts + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertToOIDCConfigCarriesJTISettings(t *testing.T) { + factory := NewProviderFactory() + + config, err := factory.convertToOIDCConfig(map[string]interface{}{ + "issuer": "https://issuer.example", + "clientId": "client", + "jtiReplayProtectionEnabled": false, + "jtiCacheMaxSize": float64(42), + }) + require.NoError(t, err) + require.NotNil(t, config.JTIReplayProtectionEnabled) + assert.False(t, *config.JTIReplayProtectionEnabled) + assert.Equal(t, 42, config.JTICacheMaxSize) + + config, err = factory.convertToOIDCConfig(map[string]interface{}{ + "issuer": "https://issuer.example", + "clientId": "client", + }) + require.NoError(t, err) + assert.Nil(t, config.JTIReplayProtectionEnabled) + assert.Zero(t, config.JTICacheMaxSize) +} diff --git a/weed/iam/sts/session_claims.go b/weed/iam/sts/session_claims.go index 8a16d55273..e7f4ce89c5 100644 --- a/weed/iam/sts/session_claims.go +++ b/weed/iam/sts/session_claims.go @@ -100,6 +100,11 @@ type STSSessionClaims struct { // guaranteed stable per OpenID Connect Core 1.0. Empty for non-federated // session types. ParentUser string `json:"puid,omitempty"` + + // SourceAccessKeyId is the access key the caller authenticated with when + // a SeaweedFS service minted the session for it (AssumeRoleForPrincipal); + // empty for every other session type. + SourceAccessKeyId string `json:"src_ak,omitempty"` } // NewSTSSessionClaims creates new STS session claims with all required information @@ -156,6 +161,8 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI ProviderIssuer: c.ProviderIssuer, RequestContext: c.RequestContext, ParentUser: c.ParentUser, + // The access key the session was minted for, if recorded + SourceAccessKeyId: c.SourceAccessKeyId, // Provide the Subject (sub) from registered claims Subject: c.Subject, Credentials: credentials, @@ -250,6 +257,12 @@ func (c *STSSessionClaims) WithSessionName(sessionName string) *STSSessionClaims return c } +// WithSourceAccessKeyId records the access key the session was minted for. +func (c *STSSessionClaims) WithSourceAccessKeyId(accessKeyId string) *STSSessionClaims { + c.SourceAccessKeyId = accessKeyId + return c +} + // WithParentUser sets the stable per-identity hash for the session. See // ComputeParentUser for the derivation rule. func (c *STSSessionClaims) WithParentUser(parentUser string) *STSSessionClaims { diff --git a/weed/iam/sts/source_access_key_test.go b/weed/iam/sts/source_access_key_test.go new file mode 100644 index 0000000000..137c045376 --- /dev/null +++ b/weed/iam/sts/source_access_key_test.go @@ -0,0 +1,42 @@ +package sts + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newSourceKeyTestSTS(t *testing.T) *STSService { + t.Helper() + s := NewSTSService() + require.NoError(t, s.Initialize(&STSConfig{ + TokenDuration: FlexibleDuration{Duration: time.Hour}, + MaxSessionLength: FlexibleDuration{Duration: 12 * time.Hour}, + Issuer: "test-issuer", + SigningKey: []byte("test-signing-key-at-least-32-bytes-long-for-security"), + })) + return s +} + +// A session minted for a caller's access key carries that key back out of its +// token; a session minted without one carries none. +func TestSessionCarriesItsSourceAccessKey(t *testing.T) { + s := newSourceKeyTestSTS(t) + resp, err := s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t-kabc", "", nil, + "iceberg-catalog", "alice", func(c *STSSessionClaims) { c.WithSourceAccessKeyId("AKIASOURCE") }) + require.NoError(t, err) + info, err := s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken) + require.NoError(t, err) + assert.Equal(t, "AKIASOURCE", info.SourceAccessKeyId) + assert.Equal(t, "alice", info.ExternalUserId) + assert.Equal(t, "iceberg-catalog", info.IdentityProvider) + + resp, err = s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t", "", nil, "iceberg-catalog", "alice") + require.NoError(t, err) + info, err = s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken) + require.NoError(t, err) + assert.Equal(t, "", info.SourceAccessKeyId) +} diff --git a/weed/iam/sts/sts_service.go b/weed/iam/sts/sts_service.go index 89d09919ad..9083b221e5 100644 --- a/weed/iam/sts/sts_service.go +++ b/weed/iam/sts/sts_service.go @@ -316,6 +316,10 @@ type SessionInfo struct { // RoleId is the unique ID of the role the session was issued for; empty // for sessions issued before role IDs were recorded. RoleId string `json:"roleId,omitempty"` + + // SourceAccessKeyId is the access key the session was minted for, when the + // minting service recorded it (AssumeRoleForPrincipal). + SourceAccessKeyId string `json:"sourceAccessKeyId,omitempty"` } // NewSTSService creates a new STS service @@ -508,8 +512,35 @@ func (s *STSService) SetIAMManagedOIDCProviders(byIssuer map[string][]ScopedOIDC } } s.iamManagedOIDCMu.Lock() + old := s.iamManagedOIDCByIssuer s.iamManagedOIDCByIssuer = cp s.iamManagedOIDCMu.Unlock() + + // Providers dropped by the swap may own background workers (the JTI + // cleanup goroutine); stop them so a replaced provider does not leak. + kept := make(map[providers.IdentityProvider]struct{}) + for _, scoped := range cp { + for _, sp := range scoped { + kept[sp.Provider] = struct{}{} + } + } + stopped := make(map[providers.IdentityProvider]struct{}) + for _, scoped := range old { + for _, sp := range scoped { + if _, ok := kept[sp.Provider]; ok { + continue + } + if _, ok := stopped[sp.Provider]; ok { + continue + } + stopped[sp.Provider] = struct{}{} + if sd, ok := sp.Provider.(interface{ Shutdown(context.Context) error }); ok { + if err := sd.Shutdown(context.Background()); err != nil { + glog.Warningf("OIDC provider %q shutdown after replacement: %v", sp.Provider.Name(), err) + } + } + } + } } // lookupOIDCProviderForAccount returns the provider that should validate @@ -583,7 +614,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass // caller passed a legacy or claim-based ARN, in which case lookup falls // back to a global (account-less) record only. roleAccountID := utils.ParseRoleARN(request.RoleArn).AccountID - externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID) + externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID, true) if err != nil { return nil, fmt.Errorf("failed to validate web identity token: %w", err) } @@ -780,7 +811,7 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass // issueSession mints temporary credentials and the self-contained JWT that // carries the whole session, shared by every assume-role entry point. func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string, - durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) { + durationSeconds *int64, providerName, subject string, options ...func(*STSSessionClaims)) (*AssumeRoleResponse, error) { sessionDuration := s.CalculateSessionDuration(durationSeconds) expiresAt := time.Now().Add(sessionDuration) @@ -810,6 +841,9 @@ func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolic if sessionPolicy != "" { sessionClaims.WithSessionPolicy(sessionPolicy) } + for _, option := range options { + option(sessionClaims) + } jwtToken, err := s.tokenGenerator.GenerateJWTWithClaims(sessionClaims) if err != nil { @@ -863,6 +897,10 @@ type AssumeRoleForPrincipalRequest struct { // DurationSeconds requests a session lifetime. DurationSeconds *int64 + + // SourceAccessKeyId optionally records the access key the caller + // authenticated with, so the session can be tied to that key. + SourceAccessKeyId string } // AssumeRoleForPrincipal issues session credentials for a caller that a @@ -908,7 +946,8 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume // Iceberg credential vending resolves no role definition, so its sessions // carry no role ID and stay bound by name. return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy, - request.DurationSeconds, request.ProviderName, request.Principal) + request.DurationSeconds, request.ProviderName, request.Principal, + func(c *STSSessionClaims) { c.WithSourceAccessKeyId(request.SourceAccessKeyId) }) } // ValidateSessionToken validates a session token and returns session information @@ -969,7 +1008,7 @@ func (s *STSService) validateAssumeRoleWithWebIdentityRequest(request *AssumeRol // validateWebIdentityToken validates the web identity token with strict issuer-to-provider mapping // SECURITY: JWT tokens with a specific issuer claim MUST only be validated by the provider for that issuer // SECURITY: This method only accepts JWT tokens. Non-JWT authentication must use AssumeRoleWithCredentials with explicit ProviderName. -func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) { +func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string, singleUse bool) (*providers.ExternalIdentity, providers.IdentityProvider, error) { // Try to extract issuer from JWT token for strict validation issuer, err := s.extractIssuerFromJWT(token) if err != nil { @@ -992,8 +1031,10 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc return nil, nil, fmt.Errorf("no identity provider registered for issuer: %s", issuer) } - // Authenticate with the correct provider for this issuer - identity, err := provider.Authenticate(ctx, token) + // Authenticate with the correct provider for this issuer. Only the STS + // exchange authenticates single-use: a bearer token is presented on every + // request, so consuming it here would reject all but the first one. + identity, err := authenticateOIDCProvider(ctx, provider, token, singleUse) if err != nil { // Map provider errors to STS errors using errors.Is() for robust error checking // This eliminates fragile string matching and provides reliable error classification @@ -1025,14 +1066,29 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc // only, then static-config); call ValidateWebIdentityTokenForAccount when the assumed-role account // is known. func (s *STSService) ValidateWebIdentityToken(ctx context.Context, token string) (*providers.ExternalIdentity, providers.IdentityProvider, error) { - return s.validateWebIdentityToken(ctx, token, "") + return s.validateWebIdentityToken(ctx, token, "", false) } // ValidateWebIdentityTokenForAccount mirrors ValidateWebIdentityToken but // scopes the IAM-managed provider lookup to roleAccountID. Pass "" for // callers that don't yet know the account (e.g. claim-based mode). func (s *STSService) ValidateWebIdentityTokenForAccount(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) { - return s.validateWebIdentityToken(ctx, token, roleAccountID) + return s.validateWebIdentityToken(ctx, token, roleAccountID, false) +} + +// authenticateOIDCProvider authenticates the token, consuming it for replay +// protection when the caller is the single-use STS exchange and the provider +// supports it. Other providers and non-exchange callers authenticate without +// consuming. +func authenticateOIDCProvider(ctx context.Context, provider providers.IdentityProvider, token string, singleUse bool) (*providers.ExternalIdentity, error) { + if singleUse { + if su, ok := provider.(interface { + AuthenticateSingleUse(context.Context, string) (*providers.ExternalIdentity, error) + }); ok { + return su.AuthenticateSingleUse(ctx, token) + } + } + return provider.Authenticate(ctx, token) } // extractIssuerFromJWT extracts the issuer (iss) claim from a JWT token without verification diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 26b2d9c606..63f85623f2 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -42,10 +42,11 @@ type IamS3ApiConfigure struct { } type IamServerOption struct { - Masters map[string]pb.ServerAddress - Filers []pb.ServerAddress - Port int - GrpcDialOption grpc.DialOption + Masters map[string]pb.ServerAddress + Filers []pb.ServerAddress + Port int + GrpcDialOption grpc.DialOption + IamSelfServiceRequiresGrant bool } type IamApiServer struct { @@ -87,8 +88,9 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli s3ApiConfigure = configure s3Option := s3api.S3ApiServerOption{ - Filers: option.Filers, - GrpcDialOption: option.GrpcDialOption, + Filers: option.Filers, + GrpcDialOption: option.GrpcDialOption, + IamSelfServiceRequiresGrant: option.IamSelfServiceRequiresGrant, } // Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified) diff --git a/weed/mount/peer_registrar_test.go b/weed/mount/peer_registrar_test.go index 7fee73d9bd..21cb7ac7f2 100644 --- a/weed/mount/peer_registrar_test.go +++ b/weed/mount/peer_registrar_test.go @@ -8,6 +8,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) // fakeFilerClient captures MountRegister/MountList calls and lets the test @@ -17,22 +18,21 @@ type fakeFilerClient struct { filer_pb.SeaweedFilerClient // embed for methods we don't need mu sync.Mutex - registerCalls []filer_pb.MountRegisterRequest + registerCalls []*filer_pb.MountRegisterRequest listResponse filer_pb.MountListResponse } func (f *fakeFilerClient) MountRegister(ctx context.Context, req *filer_pb.MountRegisterRequest, opts ...grpc.CallOption) (*filer_pb.MountRegisterResponse, error) { f.mu.Lock() defer f.mu.Unlock() - f.registerCalls = append(f.registerCalls, *req) + f.registerCalls = append(f.registerCalls, proto.Clone(req).(*filer_pb.MountRegisterRequest)) return &filer_pb.MountRegisterResponse{}, nil } func (f *fakeFilerClient) MountList(ctx context.Context, req *filer_pb.MountListRequest, opts ...grpc.CallOption) (*filer_pb.MountListResponse, error) { f.mu.Lock() defer f.mu.Unlock() - resp := f.listResponse // value copy - return &resp, nil + return proto.Clone(&f.listResponse).(*filer_pb.MountListResponse), nil } // fakeFilerFleet maps each addr to its own fakeFilerClient so a test can diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index 4fb2581581..3e9c29f813 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -157,11 +157,11 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO return ret, lastError } -func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) { +func LookupJwt(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) { - WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error { + WithMasterServerClient(ctx, false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error { - resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{ + resp, grpcErr := masterClient.LookupVolume(ctx, &master_pb.LookupVolumeRequest{ VolumeOrFileIds: []string{fileId}, }) if grpcErr != nil { diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index 1fedb74bc4..a5746c8462 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) @@ -101,6 +102,9 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri if offset > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } + if jwt != "" { + req.Header.Set("Authorization", security.BearerPrefix+jwt) + } resp, err := util_http.Do(req) if err != nil { @@ -118,7 +122,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri case http.StatusPartialContent: break default: - return written, fmt.Errorf("Read chunk needle error: [%d] %s", resp.StatusCode, fileUrl) + return written, util_http.StatusError(fileUrl, resp) } return io.Copy(w, resp.Body) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 1d94f5bab3..347b1647fd 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -10,6 +10,7 @@ import ( "path" "strconv" "strings" + "sync" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -50,6 +51,10 @@ type StoragePreference struct { type GetMasterFn func(ctx context.Context) pb.ServerAddress +// submitUploadWorkers bounds how many files one SubmitFiles call uploads at +// once; the batch assign above already fetched all their file ids. +const submitUploadWorkers = 4 + func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) { results := make([]SubmitResult, len(files)) var totalBytes int64 @@ -72,8 +77,11 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []* } ret, err := Assign(context.Background(), masterFn, grpcDialOption, ar) if err != nil { - for index := range files { + for index, file := range files { results[index].Error = err.Error() + if closer, ok := file.Reader.(io.Closer); ok { + closer.Close() + } } return results, err } @@ -87,14 +95,35 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []* file.Server = ret.PublicUrl } file.Pref = pref - results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption) - if err != nil { - results[index].Error = err.Error() - } results[index].Fid = file.Fid results[index].FileUrl = ret.PublicUrl + "/" + file.Fid } - return results, nil + + var wg sync.WaitGroup + var lastErrLock sync.Mutex + jobs := make(chan int) + for w := 0; w < min(submitUploadWorkers, len(files)); w++ { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption) + results[index].Size = size + if uploadErr != nil { + results[index].Error = uploadErr.Error() + lastErrLock.Lock() + err = uploadErr + lastErrLock.Unlock() + } + } + }() + } + for index := range files { + jobs <- index + } + close(jobs) + wg.Wait() + return results, err } func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) { @@ -118,6 +147,8 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) { fi, fiErr := fh.Stat() if fiErr != nil { glog.V(0).Info("Failed to stat file:", fullPathFilename) + fh.Close() + ret.Reader = nil return ret, fiErr } ret.ModTime = fi.ModTime().UTC().Unix() diff --git a/weed/operation/upload_chunked.go b/weed/operation/upload_chunked.go index 244a553f96..8124500dfa 100644 --- a/weed/operation/upload_chunked.go +++ b/weed/operation/upload_chunked.go @@ -9,6 +9,7 @@ import ( "fmt" "hash" "io" + "net/http" "sort" "sync" "time" @@ -45,6 +46,7 @@ type ChunkedUploadOption struct { Cipher bool // encrypt data on volume servers AssignFunc func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) UploadFunc func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) // Optional: for testing + LookupJwtFunc func(ctx context.Context, fileId string) security.EncodedJwt // Optional: renews the volume JWT for a cleanup delete } var chunkBufferPool = sync.Pool{ @@ -213,7 +215,7 @@ uploadLoop: // failed write can still leave a copy behind. No chunk will name // this fid whether we retry or give up here, and an unreferenced // needle is not garbage vacuum can find, so drop it either way. - deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt) + deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt, opt) if attempt == chunkAssignAttempts || !ShouldReassignUpload(uploadResultErr) || objectFailed() { break } @@ -413,19 +415,46 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data if firstErr != nil { // A failed fan-out records no chunk, so roll back the copies that landed // before the cancel rather than leaking them as orphans. - deleteChunkFromHolders(succeeded, fid, jwt) + deleteChunkFromHolders(succeeded, fid, jwt, opt) return nil, firstErr } return first, nil } +// cleanupJwtLookupTimeout bounds a cleanup-token renewal, covering the wait +// for a master plus the lookup RPC, so a master outage cannot stall the +// rollback of a failed fan-out. +const cleanupJwtLookupTimeout = 10 * time.Second + // deleteChunkFromHolders best-effort removes a needle from each holder it landed // on, using type=replicate so the volume drops only its local copy. A failed // delete falls back to vacuum reclaiming the orphan. -func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt) { +func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt, opt *ChunkedUploadOption) { + // An unresponsive holder can stall the upload past the write JWT's expiry, + // so renew the token whenever a holder rejects the delete as unauthorized. + // Each holder's delete is retried at most once, and a slow holder that + // outlives the renewed token can still trigger another renewal. + deleteJwt := jwt for _, host := range hosts { - if err := util_http.Delete(fmt.Sprintf("http://%s/%s?type=replicate", host, fid), string(jwt)); err != nil { + deleteUrl := fmt.Sprintf("http://%s/%s?type=replicate", host, fid) + body, status, err := util_http.DeleteProxied(deleteUrl, string(deleteJwt)) + if status == http.StatusUnauthorized && opt.LookupJwtFunc != nil { + lookupCtx, cancel := context.WithTimeout(context.Background(), cleanupJwtLookupTimeout) + token := opt.LookupJwtFunc(lookupCtx, fid) + cancel() + if token != "" { + deleteJwt = token + body, status, err = util_http.DeleteProxied(deleteUrl, string(deleteJwt)) + } + } + if err != nil { glog.Warningf("replica fan-out cleanup: delete %s from %s: %v", fid, host, err) + continue + } + switch status { + case http.StatusNotFound, http.StatusNoContent, http.StatusAccepted, http.StatusOK: + default: + glog.Warningf("replica fan-out cleanup: delete %s from %s: status %d: %s", fid, host, status, body) } } } diff --git a/weed/operation/upload_chunked_test.go b/weed/operation/upload_chunked_test.go index b1accb3b75..87ab218ebc 100644 --- a/weed/operation/upload_chunked_test.go +++ b/weed/operation/upload_chunked_test.go @@ -26,13 +26,11 @@ func TestUploadReaderInChunksReturnsPartialResultsOnError(t *testing.T) { testData := bytes.Repeat([]byte("test data for chunk upload failure testing"), 1000) // ~40KB reader := bytes.NewReader(testData) - uploadAttempts := 0 + var uploadAttempts int32 // Create a mock assign function that succeeds for first chunk, then fails assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) { - uploadAttempts++ - - if uploadAttempts == 1 { + if atomic.AddInt32(&uploadAttempts, 1) == 1 { // First chunk succeeds return nil, &AssignResult{ Fid: "test-fid-1,1234", @@ -679,3 +677,90 @@ func TestUploadReaderInChunksAppendsFsyncWhenAssigned(t *testing.T) { t.Errorf("Expected the upload URL to carry ?fsync=true when the assignment says fsync, got %q", gotUploadUrl) } } + +// A holder that stalls past the assignment JWT's expiry answers the rollback +// delete with 401; the cleanup renews the token instead of leaking the copy. +func TestDeleteChunkFromHoldersRenewsExpiredJwt(t *testing.T) { + var staleDeletes, freshDeletes int32 + holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusCreated) + return + } + switch r.Header.Get("Authorization") { + case "Bearer fresh-jwt": + atomic.AddInt32(&freshDeletes, 1) + w.WriteHeader(http.StatusAccepted) + default: + atomic.AddInt32(&staleDeletes, 1) + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"error":"wrong jwt"}`) + } + })) + defer holder.Close() + + lookups := 0 + deleteChunkFromHolders( + []string{strings.TrimPrefix(holder.URL, "http://")}, + "1,0a0b0c0d", + "stale-jwt", + &ChunkedUploadOption{ + LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt { + lookups++ + return "fresh-jwt" + }, + }, + ) + + if lookups != 1 { + t.Fatalf("expected one jwt renewal, got %d", lookups) + } + if atomic.LoadInt32(&freshDeletes) != 1 { + t.Fatalf("expected the delete retried with the fresh jwt, got %d fresh deletes", freshDeletes) + } +} + +// A later holder that rejects the renewed token can trigger another renewal: +// the one-retry limit is per holder, not per cleanup pass. +func TestDeleteChunkFromHoldersRenewsPerHolder(t *testing.T) { + var acceptedDeletes int32 + holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusCreated) + return + } + if r.Header.Get("Authorization") == "Bearer jwt-2" { + atomic.AddInt32(&acceptedDeletes, 1) + w.WriteHeader(http.StatusAccepted) + return + } + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"error":"wrong jwt"}`) + })) + defer holder.Close() + + tokens := []security.EncodedJwt{"jwt-1", "jwt-2"} + lookups := 0 + deleteChunkFromHolders( + []string{ + strings.TrimPrefix(holder.URL, "http://"), + strings.TrimPrefix(holder.URL, "http://"), + }, + "1,0a0b0c0d", + "stale-jwt", + &ChunkedUploadOption{ + LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt { + token := tokens[lookups] + lookups++ + return token + }, + }, + ) + + if lookups != 2 { + t.Fatalf("expected a renewal per rejecting holder, got %d", lookups) + } + if atomic.LoadInt32(&acceptedDeletes) != 1 { + t.Fatalf("expected the second holder's retried delete to land, got %d", acceptedDeletes) + } +} diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index a3903cff48..257349b021 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -67,6 +67,16 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.Fu // consumers would mistake for an entry. const ListSnapshotTsNsTrailerKey = "sw-list-snapshot-ts-ns" +// TraverseModeHeaderKey is gRPC metadata on a TraverseBfsMetadata stream. As +// request metadata, TraverseModeSnapshot asks for the entries from one store +// snapshot (any order, entry_ts_ns left 0) instead of a breadth-first walk; as +// a response header it says what the filer did. An older filer sends none. +const ( + TraverseModeHeaderKey = "sw-traverse-mode" + TraverseModeSnapshot = "snapshot" + TraverseModeBfs = "bfs" +) + type EachEntryFunction func(entry *Entry, isLast bool) error func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (err error) { diff --git a/weed/plugin/worker/worker.go b/weed/plugin/worker/worker.go index 5c270bfa7e..28a04c2f2c 100644 --- a/weed/plugin/worker/worker.go +++ b/weed/plugin/worker/worker.go @@ -660,8 +660,7 @@ func (w *Worker) buildHeartbeat() *plugin_pb.WorkerHeartbeat { if work == nil { continue } - cloned := *work - running = append(running, &cloned) + running = append(running, proto.Clone(work).(*plugin_pb.RunningWork)) } w.runningMu.RUnlock() diff --git a/weed/remote_storage/s3/s3_storage_client.go b/weed/remote_storage/s3/s3_storage_client.go index 7664407ebc..9641195ea0 100644 --- a/weed/remote_storage/s3/s3_storage_client.go +++ b/weed/remote_storage/s3/s3_storage_client.go @@ -168,6 +168,27 @@ func S3CompatibleEndpoint(conf *remote_pb.RemoteConf) (string, bool) { return opt.endpoint, true } +// awsCredentialSourceConfigured reports whether the environment names a source +// for the AWS SDK default credential chain, such as static keys, a profile, +// IRSA web identity or the ECS/EKS container credentials endpoint. +func awsCredentialSourceConfigured() bool { + for _, name := range []string{ + "AWS_ACCESS_KEY_ID", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + } { + if os.Getenv(name) != "" { + return true + } + } + return false +} + // MakeWithHTTPClient builds the client for any S3-SDK-backed remote storage // type using the supplied *http.Client (or the AWS SDK default when nil). // Callers that need to pin the dial path against DNS rebinding can pass a @@ -191,12 +212,32 @@ func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (re } if opt.accessKey != "" && opt.secretKey != "" { config.Credentials = credentials.NewStaticCredentials(opt.accessKey, opt.secretKey, "") - } else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" { - // Explicitly disable signing for public buckets. + } else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" && !awsCredentialSourceConfigured() { + // Explicitly disable signing for public buckets. When the environment + // names an AWS credential source, leave Credentials nil so the SDK + // default chain (env, web identity, container, shared profile) is used. config.Credentials = credentials.AnonymousCredentials + } else if config.Credentials == nil && config.HTTPClient != nil { + // Remote credential providers (container endpoints, IMDS, web + // identity) fetch through this HTTP client; the guarded one is for the + // S3 endpoint and blocks the link-local credential endpoints, so the + // default chain resolves on an unguarded client. Shared config stays + // enabled so web identity and profiles resolve here too, and the + // region carries over so STS can sign assume-role requests. + credSess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{Region: config.Region, HTTPClient: &http.Client{}}, + SharedConfigState: session.SharedConfigEnable, + }) + if err != nil { + return nil, fmt.Errorf("create %s credential session: %w", opt.name, err) + } + config.Credentials = credSess.Config.Credentials } - sess, err := session.NewSession(config) + sess, err := session.NewSessionWithOptions(session.Options{ + Config: *config, + SharedConfigState: session.SharedConfigEnable, + }) if err != nil { return nil, fmt.Errorf("create %s session: %w", opt.name, err) } diff --git a/weed/remote_storage/s3/s3_storage_client_test.go b/weed/remote_storage/s3/s3_storage_client_test.go index 568e4a6333..dc404d5b18 100644 --- a/weed/remote_storage/s3/s3_storage_client_test.go +++ b/weed/remote_storage/s3/s3_storage_client_test.go @@ -21,6 +21,7 @@ import ( ) func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) { + clearAWSCredentialEnv(t) maker := s3RemoteStorageMaker{} conf := &remote_pb.RemoteConf{ Type: "s3", @@ -40,6 +41,74 @@ func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) { require.Same(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) } +func clearAWSCredentialEnv(t *testing.T) { + t.Helper() + for _, name := range []string{ + "AWS_ACCESS_KEY_ID", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + } { + t.Setenv(name, "") + } +} + +func TestS3MakeUsesDefaultCredentialChainWhenKeysAreEmptyAndSourceIsConfigured(t *testing.T) { + for name, value := range map[string]string{ + "AWS_ACCESS_KEY_ID": "env-access", + "AWS_PROFILE": "some-profile", + "AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token", + "AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:1/creds", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/creds", + } { + t.Run(name, func(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv(name, value) + t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/test") + + remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) + }) + } +} + +// A caller that pins a custom http.Client still gets the default chain, so +// remote providers resolve on an unguarded client while S3 requests use the +// pinned one. +func TestS3MakeResolvesEnvCredentialsWithCustomHTTPClient(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv("AWS_ACCESS_KEY_ID", "env-access") + t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret") + + remoteClient, err := MakeWithHTTPClient(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }, &http.Client{}) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials) + value, err := s3Client.Config.Credentials.Get() + require.NoError(t, err) + require.Equal(t, "env-access", value.AccessKeyID) +} + func TestS3MakeUsesStaticCredentialsWhenKeysAreProvided(t *testing.T) { maker := s3RemoteStorageMaker{} conf := &remote_pb.RemoteConf{ @@ -396,3 +465,24 @@ func TestIsNoSuchKey(t *testing.T) { require.True(t, isNoSuchKey(fmt.Errorf("download: %w", awserr.New(awss3.ErrCodeNoSuchKey, "missing", nil)))) require.False(t, isNoSuchKey(awserr.New(awss3.ErrCodeNoSuchBucket, "missing", nil))) } + +func TestS3MakeResolvesEnvCredentialsWhenAnonymousDefault(t *testing.T) { + clearAWSCredentialEnv(t) + t.Setenv("AWS_ACCESS_KEY_ID", "env-access") + t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret") + + remoteClient, err := s3RemoteStorageMaker{}.Make(&remote_pb.RemoteConf{ + Type: "s3", + S3Region: "us-east-1", + S3Endpoint: "http://localhost:8333", + S3ForcePathStyle: true, + }) + require.NoError(t, err) + + s3Client, ok := remoteClient.(*s3RemoteStorageClient).conn.(*awss3.S3) + require.True(t, ok) + value, err := s3Client.Config.Credentials.Get() + require.NoError(t, err) + require.Equal(t, "env-access", value.AccessKeyID) + require.Equal(t, "env-secret", value.SecretAccessKey) +} diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index bffce98b9e..2675975dd8 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -229,8 +229,7 @@ func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk manifestDataChunks := make([]*filer_pb.FileChunk, len(replicatedResolvedChunks)) for i, chunk := range replicatedResolvedChunks { - copied := *chunk - manifestDataChunks[i] = &copied + manifestDataChunks[i] = proto.Clone(chunk).(*filer_pb.FileChunk) } filer_pb.BeforeEntrySerialization(manifestDataChunks) manifestData, err := proto.Marshal(&filer_pb.FileChunkManifest{ diff --git a/weed/s3api/auth_access_key_attribution_test.go b/weed/s3api/auth_access_key_attribution_test.go new file mode 100644 index 0000000000..729e50e98a --- /dev/null +++ b/weed/s3api/auth_access_key_attribution_test.go @@ -0,0 +1,75 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +const ( + sharedKeyA = "AKIASHAREDKEYAAAAAAA" + sharedSecretA = "secret-a-wJalrXUtnFEMI/K7MDENG/bPxRfiCY" + sharedKeyB = "AKIASHAREDKEYBBBBBBB" + sharedSecretB = "secret-b-wJalrXUtnFEMI/K7MDENG/bPxRfiCY" +) + +// newSharedIdentityIAM returns one identity holding two key pairs, as when +// several people or services are given their own key on one identity. +func newSharedIdentityIAM() *IdentityAccessManagement { + iam := &IdentityAccessManagement{} + shared := &Identity{ + Name: "shared-user", + Credentials: []*Credential{ + {AccessKey: sharedKeyA, SecretKey: sharedSecretA}, + {AccessKey: sharedKeyB, SecretKey: sharedSecretB}, + }, + Actions: []Action{s3_constants.ACTION_READ}, + } + iam.identities = []*Identity{shared} + iam.accessKeyIdent = map[string]*Identity{sharedKeyA: shared, sharedKeyB: shared} + iam.nameToIdentity = map[string]*Identity{shared.Name: shared} + return iam +} + +// Two keys of one identity: the audit entry names the same requester for both +// and tells them apart by the key each request was signed with. +func TestAuditLogCarriesTheSigningAccessKey(t *testing.T) { + iam := newSharedIdentityIAM() + for _, key := range []struct{ id, secret string }{{sharedKeyA, sharedSecretA}, {sharedKeyB, sharedSecretB}} { + req, err := newTestRequest(http.MethodGet, "https://example.com/bucket/ns/t/data/f.parquet", 0, nil) + require.NoError(t, err) + require.NoError(t, signRequestV4(req, key.id, key.secret)) + // track() installs the holder before authentication runs. + outer := s3_constants.EnsureIdentityHolder(req) + + identity, errCode := iam.reqSignatureV4Verify(outer) + require.Equal(t, s3err.ErrNone, errCode) + iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrNone, func(http.ResponseWriter, *http.Request) {}) + + log := s3err.GetAccessLog(outer, http.StatusOK, s3err.ErrNone) + assert.Equal(t, "shared-user", log.Requester) + assert.Equal(t, key.id, log.AccessKeyId, "audit entry must name the key the request was signed with") + } +} + +// A request refused after authentication still names its key, like its requester. +func TestAuditLogCarriesTheAccessKeyOnTheDenyPath(t *testing.T) { + iam := newSharedIdentityIAM() + req, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + require.NoError(t, signRequestV4(req, sharedKeyB, sharedSecretB)) + outer := s3_constants.EnsureIdentityHolder(req) + identity, errCode := iam.reqSignatureV4Verify(outer) + require.Equal(t, s3err.ErrNone, errCode) + + iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrAccessDenied, func(http.ResponseWriter, *http.Request) { + t.Fatal("handler must not run on the deny path") + }) + assert.Equal(t, sharedKeyB, s3err.GetAccessLog(outer, http.StatusForbidden, s3err.ErrAccessDenied).AccessKeyId) +} diff --git a/weed/s3api/auth_access_key_post_policy_test.go b/weed/s3api/auth_access_key_post_policy_test.go new file mode 100644 index 0000000000..50e36bb023 --- /dev/null +++ b/weed/s3api/auth_access_key_post_policy_test.go @@ -0,0 +1,86 @@ +package s3api + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// signedPostFormV2 builds a multipart POST Object request whose policy is +// signed with SigV2 under postAuthzAccessKey and secret. +func signedPostFormV2(t *testing.T, key, secret string) *http.Request { + t.Helper() + policyJSON := fmt.Sprintf(`{"expiration":"%s","conditions":[["eq","$bucket","%s"],["eq","$key","%s"]]}`, + time.Now().UTC().Add(time.Hour).Format("2006-01-02T15:04:05.000Z"), postAuthzBucket, key) + encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON)) + + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + for _, field := range [][2]string{{"bucket", postAuthzBucket}, {"key", key}, {"AWSAccessKeyId", postAuthzAccessKey}, + {"policy", encodedPolicy}, {"Signature", calculateSignatureV2(encodedPolicy, secret)}} { + require.NoError(t, writer.WriteField(field[0], field[1])) + } + filePart, err := writer.CreateFormFile("file", "payload.txt") + require.NoError(t, err) + _, err = filePart.Write([]byte("payload")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + return mux.SetURLVars(req, map[string]string{"bucket": postAuthzBucket}) +} + +// A browser POST upload is verified in the handler from its form fields; the +// key of the verified form reaches the audit log like a signed PUT's. The +// bucket policy denies denied/ so the request ends before the filer. +func TestPostPolicyUploadRecordsTheSigningKey(t *testing.T) { + s3a := newPostAuthzServer(t) + for name, req := range map[string]*http.Request{ + "SigV4": newSignedPostRequest(t, "denied/secret.txt"), + "SigV2": signedPostFormV2(t, "denied/secret.txt", postAuthzSecretKey), + } { + outer := s3_constants.EnsureIdentityHolder(req) + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, outer) + require.Equal(t, http.StatusForbidden, rec.Code, "%s: %s", name, rec.Body.String()) + assert.Equal(t, postAuthzAccessKey, s3err.GetAccessLog(outer, http.StatusForbidden, s3err.ErrAccessDenied).AccessKeyId, name) + } + + // A form whose signature does not verify records nothing. + outer := s3_constants.EnsureIdentityHolder(signedPostFormV2(t, "denied/secret.txt", "not-the-secret")) + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, outer) + require.Contains(t, rec.Body.String(), "SignatureDoesNotMatch") + assert.Equal(t, "", s3_constants.GetAccessKeyIdFromContext(outer)) +} + +// A signature that verifies for one virtual-hosted bucket but is routed to +// another is refused after verification; the refusal still names the key. +func TestSignedVirtualHostRetargetRecordsTheSigningKey(t *testing.T) { + iam := newVirtualHostTestIAM("s3.test") + r, err := newTestRequest("GET", "http://alpha-bkt.s3.test/shared.txt", 0, nil) + require.NoError(t, err) + r.Header.Set("Host", "alpha-bkt.s3.test") + signV4WithPath(r, "access_key_1", "secret_key_1", r.URL.Path) + r.Host = "beta-bkt.s3.test" + r.Header.Set("X-Forwarded-Host", "alpha-bkt.s3.test") + r = mux.SetURLVars(r, map[string]string{"bucket": "beta-bkt", "object": "shared.txt"}) + r = s3_constants.EnsureIdentityHolder(r) + + _, _, errCode := iam.doesSignatureMatch(r) + require.Equal(t, s3err.ErrAccessDenied, errCode) + assert.Equal(t, "access_key_1", s3_constants.GetAccessKeyIdFromContext(r)) +} diff --git a/weed/s3api/auth_access_key_shapes_test.go b/weed/s3api/auth_access_key_shapes_test.go new file mode 100644 index 0000000000..5ea2b2ac4c --- /dev/null +++ b/weed/s3api/auth_access_key_shapes_test.go @@ -0,0 +1,141 @@ +package s3api + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// The key recorded for a request is the key of the credential that verified, +// whatever else the request carries. A request names its key in several +// places (SigV4 header, X-Amz-Credential, SigV2 header, AWSAccessKeyId); +// getRequestAuthType picks one and that verifier records it, so a second, +// unverified key in another place never reaches the audit log or a vended +// session. + +func authAndRecordedKey(t *testing.T, iam *IdentityAccessManagement, r *http.Request) (s3err.ErrorCode, string) { + t.Helper() + r = s3_constants.EnsureIdentityHolder(r) + _, errCode, _ := iam.authenticateRequestInternal(r) + return errCode, s3_constants.GetAccessKeyIdFromContext(r) +} + +func credOf(key, secret string) *Credential { return &Credential{AccessKey: key, SecretKey: secret} } + +func TestRecordedKeyIsTheVerifiedKeyForSigV4Header(t *testing.T) { + iam := newSharedIdentityIAM() + r, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + require.NoError(t, signRequestV4(r, sharedKeyA, sharedSecretA)) + errCode, got := authAndRecordedKey(t, iam, r) + require.Equal(t, s3err.ErrNone, errCode) + assert.Equal(t, sharedKeyA, got) +} + +// SigV2 header signed with A, plus X-Amz-Credential naming B or garbage: the +// request authenticates as A (SigV2 is checked first) and A is recorded. +func TestRecordedKeyIgnoresAJunkCredentialQueryOnASigV2Request(t *testing.T) { + iam := newSharedIdentityIAM() + for _, junk := range []string{sharedKeyB + "/20261016/us-east-1/s3/aws4_request", "GARBAGE"} { + r, err := http.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet?X-Amz-Credential="+url.QueryEscape(junk), nil) + require.NoError(t, err) + r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) + r.Header.Set("Authorization", signatureV2(credOf(sharedKeyA, sharedSecretA), r.Method, r.URL.EscapedPath(), r.URL.Query().Encode(), r.Header)) + errCode, got := authAndRecordedKey(t, iam, r) + require.Equal(t, s3err.ErrNone, errCode, "the SigV2 signature with A verifies (junk %q)", junk) + assert.Equal(t, sharedKeyA, got, "the key that verified, not the junk %q", junk) + } +} + +// SigV2 presigned with A, plus a junk SigV4 Authorization naming B: the +// presigned A verifies and A is recorded. +func TestRecordedKeyIgnoresAJunkSigV4HeaderOnASigV2PresignedRequest(t *testing.T) { + iam := newSharedIdentityIAM() + expires := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10) + q := url.Values{"AWSAccessKeyId": {sharedKeyA}, "Expires": {expires}} + r, err := http.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet?"+q.Encode(), nil) + require.NoError(t, err) + sig := preSignatureV2(credOf(sharedKeyA, sharedSecretA), r.Method, r.URL.EscapedPath(), r.URL.Query().Encode(), r.Header, expires) + q.Set("Signature", sig) + r.URL.RawQuery = q.Encode() + r.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+sharedKeyB+"/20261016/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=00") + + errCode, got := authAndRecordedKey(t, iam, r) + require.Equal(t, s3err.ErrNone, errCode, "the SigV2 presigned A verifies (SigV2 is dispatched first)") + assert.Equal(t, sharedKeyA, got, "the presigned key A is recorded, never the unverified B") +} + +// A refused signature records nothing, even when the key exists. +func TestRecordedKeyIsEmptyWhenTheSignatureFails(t *testing.T) { + iam := newSharedIdentityIAM() + r, err := newTestRequest(http.MethodGet, "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + require.NoError(t, signRequestV4(r, sharedKeyA, "not-the-secret")) + errCode, got := authAndRecordedKey(t, iam, r) + require.NotEqual(t, s3err.ErrNone, errCode) + assert.Equal(t, "", got) + + r2 := httptest.NewRequest(http.MethodGet, "http://example.com/bucket/f.parquet", nil) + r2.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) + r2.Header.Set("Authorization", "AWS "+sharedKeyB+":bm90LWEtc2lnbmF0dXJl") + errCode, got = authAndRecordedKey(t, iam, r2) + require.NotEqual(t, s3err.ErrNone, errCode) + assert.Equal(t, "", got) +} + +// A streaming (aws-chunked) upload records the key of its seed signature. +func TestRecordedKeyForAStreamingUpload(t *testing.T) { + iam := newSharedIdentityIAM() + now := time.Now().UTC() + amzDate, dateStamp := now.Format(iso8601Format), now.Format(yyyymmdd) + scope := dateStamp + "/us-east-1/s3/aws4_request" + hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + canonicalHeaders := "content-encoding:aws-chunked\n" + "host:s3.amazonaws.com\n" + + "x-amz-content-sha256:" + hashedPayload + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-decoded-content-length:4\n" + signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length" + canonicalRequest := "PUT\n/bucket/ns/t/data/f.parquet\n\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + hashedPayload + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + getSHA256Hash([]byte(canonicalRequest)) + seed := getSignature(getSigningKey(sharedSecretB, dateStamp, "us-east-1", "s3"), stringToSign) + + r, err := http.NewRequest(http.MethodPut, "http://s3.amazonaws.com/bucket/ns/t/data/f.parquet", bytes.NewReader([]byte("ignored"))) + require.NoError(t, err) + r.Header.Set("Host", "s3.amazonaws.com") + r.Header.Set("x-amz-date", amzDate) + r.Header.Set("x-amz-content-sha256", hashedPayload) + r.Header.Set("Content-Encoding", "aws-chunked") + r.Header.Set("x-amz-decoded-content-length", "4") + r.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", sharedKeyB, scope, signedHeaders, seed)) + + errCode, got := authAndRecordedKey(t, iam, r) + require.Equal(t, s3err.ErrNone, errCode) + assert.Equal(t, sharedKeyB, got) +} + +// An STS session's request records the session's temporary key. +func TestRecordedKeyForAnSTSSession(t *testing.T) { + stsService, config := setupTestSTSService(t) + iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory") + iam.SetIAMIntegration(&S3IAMIntegration{stsService: stsService, enabled: true}) + sessionInfo, sessionToken := issueTestSession(t, stsService, config) + + r, err := newTestRequest(http.MethodGet, "https://example.com/reports/data.csv", 0, nil) + require.NoError(t, err) + r.Header.Set("X-Amz-Security-Token", sessionToken) + require.NoError(t, signRequestV4(r, sessionInfo.Credentials.AccessKeyId, sessionInfo.Credentials.SecretAccessKey)) + outer := s3_constants.EnsureIdentityHolder(r) + identity, errCode := iam.reqSignatureV4Verify(outer) + require.Equal(t, s3err.ErrNone, errCode) + iam.handleAuthResult(httptest.NewRecorder(), outer, identity, s3err.ErrNone, func(http.ResponseWriter, *http.Request) {}) + assert.Equal(t, sessionInfo.Credentials.AccessKeyId, s3err.GetAccessLog(outer, http.StatusOK, s3err.ErrNone).AccessKeyId) +} diff --git a/weed/s3api/auth_copy_source_test.go b/weed/s3api/auth_copy_source_test.go index 746f16009f..21415ddc9e 100644 --- a/weed/s3api/auth_copy_source_test.go +++ b/weed/s3api/auth_copy_source_test.go @@ -3,6 +3,7 @@ package s3api import ( "context" "net/http" + "net/url" "testing" "github.com/stretchr/testify/assert" @@ -251,9 +252,10 @@ func TestAuthorizeCopySource_PresignedURLSessionTokenPreserved(t *testing.T) { } // Presigned URL: session token is in the query string, no - // X-Amz-Security-Token header. The destination URL also carries unrelated - // SigV4 query params (X-Amz-Algorithm, etc.) that must not leak into the - // synthetic source action resolution. + // X-Amz-Security-Token header. The destination URL also carries the rest of + // the SigV4 presign params, which ride along so auth-derived policy + // conditions (s3:authType) match the real request; none of them steer + // ResolveS3Action. req, err := http.NewRequest(http.MethodPut, "http://s3.local/bucket-a/dst.bin?X-Amz-Algorithm=AWS4-HMAC-SHA256"+ "&X-Amz-Credential=stsKey%2F20260518%2Fus-east-1%2Fs3%2Faws4_request"+ @@ -267,8 +269,12 @@ func TestAuthorizeCopySource_PresignedURLSessionTokenPreserved(t *testing.T) { assert.Equal(t, s3err.ErrNone, errCode) assert.Equal(t, "presigned-session-token", capturedSessionToken, "session token from presigned-URL query must reach the IAM check") - // versionId omitted, only the token should be present in the synthetic query. - assert.Equal(t, "X-Amz-Security-Token=presigned-session-token", capturedRawQuery) + capturedQuery, err := url.ParseQuery(capturedRawQuery) + require.NoError(t, err) + assert.Equal(t, "presigned-session-token", capturedQuery.Get("X-Amz-Security-Token")) + assert.Equal(t, "AWS4-HMAC-SHA256", capturedQuery.Get("X-Amz-Algorithm")) + assert.False(t, capturedQuery.Has("versionId")) + assert.False(t, capturedQuery.Has("acl")) } // TestAuthorizeCopySource_PreservesCopySourceHeader verifies that the synthetic diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index bb05bb5335..46ff18366b 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -69,6 +69,11 @@ type IdentityAccessManagement struct { // IAM Integration for advanced features iamIntegration IAMIntegration + // selfServiceRequiresGrant makes the self-service IAM actions (an + // identity's own access keys, GetUser on itself) need an iam: + // grant like every other IAM action. Off by default. + selfServiceRequiresGrant bool + // Serializes resyncs of the policy set into the advanced IAM manager so that // concurrent policy updates can't apply stale views out of order. See // resyncIAMManagerPolicies. @@ -337,6 +342,7 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient hashCounters: make(map[string]*int32), filerClient: filerClient, } + iam.selfServiceRequiresGrant = option.IamSelfServiceRequiresGrant // Always initialize credential manager with fallback to defaults credentialManager, err := credential.NewCredentialManagerWithDefaults(credential.CredentialStoreTypeName(explicitStore)) @@ -3061,11 +3067,21 @@ func (iam *IdentityAccessManagement) bucketsNamedByAttachedPolicies(r *http.Requ // its query describe. versionId, when set, and the STS session token ride along // for policy conditions. Returns ErrNone when auth is disabled (checked by the // callers) or the identity is an admin. +// authQueryParams are the query keys carrying a request's authentication +// material (SigV2 and SigV4 presign parameters). None steer ResolveS3Action, +// so they can ride along on a synthesized request. +var authQueryParams = []string{ + "AWSAccessKeyId", "Signature", "Expires", + "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", "X-Amz-Expires", + "X-Amz-Signature", "X-Amz-SignedHeaders", "X-Amz-Security-Token", +} + // objectKeyPolicyRequest synthesizes a // request // for policy evaluation against a key the real URL does not name. The query is // built from scratch so a param on the real request (delete, uploadId, // partNumber, ...) cannot steer ResolveS3Action off the intended action; the -// STS token and versionId ride along for policy conditions. +// versionId and the auth-material params ride along so auth-derived policy +// conditions (s3:authType) evaluate as they did on the real request. func objectKeyPolicyRequest(r *http.Request, method, bucket, objectKey, versionId string) *http.Request { keyReq := new(http.Request) *keyReq = *r @@ -3078,9 +3094,9 @@ func objectKeyPolicyRequest(r *http.Request, method, bucket, objectKey, versionI if versionId != "" { keyQuery.Set("versionId", versionId) } - if strings.Contains(r.URL.RawQuery, "X-Amz-Security-Token") { - if token := r.URL.Query().Get("X-Amz-Security-Token"); token != "" { - keyQuery.Set("X-Amz-Security-Token", token) + for _, key := range authQueryParams { + if vals, ok := r.URL.Query()[key]; ok { + keyQuery[key] = vals } } if len(keyQuery) > 0 { diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index 72fdf9b421..3bda791e64 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -18,6 +18,7 @@ package s3api import ( + "context" "crypto/hmac" "crypto/sha1" "crypto/subtle" @@ -29,6 +30,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) @@ -67,7 +69,7 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide return iam.doesPresignV2SignatureMatch(r) } -func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) (*Identity, s3err.ErrorCode) { +func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) { accessKey := formValues.Get("AWSAccessKeyId") if accessKey == "" { @@ -107,6 +109,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http. if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) { return nil, s3err.ErrSignatureDoesNotMatch } + s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) return identity, s3err.ErrNone } @@ -164,6 +167,7 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity if !compareSignatureV2(v2Signature, expectedV2Signature) { return nil, s3err.ErrSignatureDoesNotMatch } + s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) return identity, s3err.ErrNone } @@ -222,6 +226,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request if !compareSignatureV2(signature, expectedSignature) { return nil, s3err.ErrSignatureDoesNotMatch } + s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 2289d7f77d..b0ec7fd292 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -19,6 +19,7 @@ package s3api import ( "bytes" + "context" "crypto/hmac" "crypto/sha256" "crypto/subtle" @@ -407,6 +408,10 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh } if matchedHost != "" { + // The key of the credential that verified: the audit log and the + // catalog's vended sessions name it (never a key re-read from the request). + // Recorded before the retarget check so that refusal names it too. + s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) if signedBucket, ok := bucketFromVirtualHost(matchedHost, iam.domain); ok { if routedBucket, _ := s3_constants.GetBucketAndObject(r); routedBucket != "" && routedBucket != signedBucket { glog.V(2).Infof("reject %s %s: signed host %q implies bucket %q but routed to %q", @@ -830,7 +835,7 @@ func parseSignedHeaderList(signedHeadersValue string) ([]string, s3err.ErrorCode return signedHeaders, s3err.ErrNone } -func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) (*Identity, s3err.ErrorCode) { +func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) { // Parse credential tag. credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential")) @@ -868,6 +873,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http. if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { return nil, s3err.ErrSignatureDoesNotMatch } + s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go index e946dbb306..03b37215a1 100644 --- a/weed/s3api/chunked_reader_v4_test.go +++ b/weed/s3api/chunked_reader_v4_test.go @@ -111,7 +111,7 @@ func TestNewSignV4ChunkedReaderStreamingUnsignedPayloadTrailer(t *testing.T) { runWithRequest(iam, req, t, expectedPayload) } -func runWithRequest(iam IdentityAccessManagement, req *http.Request, t *testing.T, expectedPayload string) { +func runWithRequest(iam *IdentityAccessManagement, req *http.Request, t *testing.T, expectedPayload string) { reader, errCode := iam.newChunkedReader(req) assert.NotNil(t, reader) assert.Equal(t, s3err.ErrNone, errCode) @@ -124,11 +124,11 @@ func runWithRequest(iam IdentityAccessManagement, req *http.Request, t *testing. assert.Equal(t, expectedPayload, string(data)) } -func setupIam() IdentityAccessManagement { +func setupIam() *IdentityAccessManagement { // Create an IdentityAccessManagement instance // Add default access keys and secrets - iam := IdentityAccessManagement{ + iam := &IdentityAccessManagement{ identities: []*Identity{}, accessKeyIdent: map[string]*Identity{}, accounts: map[string]*Account{}, diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 5cb0e8a833..6f22429fb6 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -2,6 +2,7 @@ package s3api import ( "cmp" + "container/heap" "context" "crypto/md5" "crypto/rand" @@ -1292,7 +1293,15 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput glog.V(2).Infof("listMultipartUploads input %v", input) - entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", "", false, math.MaxInt32) + uploadsFolder := s3a.genUploadsFolder(*input.Bucket) + var page *multipartUploadPage + err := listRetry(uploadsFolder, func() error { + page = newMultipartUploadPage(input) + return filer_pb.List(context.Background(), s3a, uploadsFolder, "", func(entry *filer_pb.Entry, isLastEntry bool) error { + page.add(entry) + return nil + }, "", false, math.MaxInt32) + }) if err != nil { // A missing .uploads folder normally lists as empty with no error; a // store that reports it as not-found still means an empty list. @@ -1304,55 +1313,104 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput return nil, s3err.ErrInternalError } - return selectMultipartUploads(input, entries), s3err.ErrNone + return page.result(), s3err.ErrNone } // selectMultipartUploads orders uploads by key, then by upload id, and resumes // after the (key-marker, upload-id-marker) position, so an aborted marker // upload does not hide the rest of its key. func selectMultipartUploads(input *s3.ListMultipartUploadsInput, entries []*filer_pb.Entry) *ListMultipartUploadsResult { + page := newMultipartUploadPage(input) + for _, entry := range entries { + page.add(entry) + } + return page.result() +} + +// multipartUploadPage accumulates one page of a ListMultipartUploads result. +// Upload order is the object key kept in extended attributes, not the upload +// id the filer sorts by, so every entry has to be seen; keeping only the +// maxUploads smallest candidates bounds the heap by the page, not the folder. +type multipartUploadPage struct { + input *s3.ListMultipartUploadsInput + limit int + top candidateHeap + more bool +} + +type multipartCandidate struct { + key string + entry *filer_pb.Entry +} + +func compareMultipartCandidate(a, b multipartCandidate) int { + return cmp.Or(strings.Compare(a.key, b.key), strings.Compare(a.entry.Name, b.entry.Name)) +} + +// candidateHeap is a max-heap on (key, upload id): the root sorts last of the +// candidates held, so popping it keeps the page's smallest entries. +type candidateHeap []multipartCandidate + +func (h candidateHeap) Len() int { return len(h) } +func (h candidateHeap) Less(i, j int) bool { + return compareMultipartCandidate(h[i], h[j]) > 0 +} +func (h candidateHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *candidateHeap) Push(x any) { *h = append(*h, x.(multipartCandidate)) } +func (h *candidateHeap) Pop() any { + old := *h + n := len(old) + *h = old[:n-1] + return old[n-1] +} + +func newMultipartUploadPage(input *s3.ListMultipartUploadsInput) *multipartUploadPage { + limit := maxUploadsList + if *input.MaxUploads < int64(limit) { + limit = int(*input.MaxUploads) + } + return &multipartUploadPage{input: input, limit: limit} +} + +func (p *multipartUploadPage) add(entry *filer_pb.Entry) { + if p.limit <= 0 || !isMultipartUploadEntry(entry) { + return + } + input := p.input + key := *objectKey(aws.String(string(entry.Extended[s3_constants.ExtMultipartObjectKey]))) + if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) { + return + } + if *input.KeyMarker != "" && (key < *input.KeyMarker || key == *input.KeyMarker && (*input.UploadIdMarker == "" || entry.Name <= *input.UploadIdMarker)) { + return + } + c := multipartCandidate{key: key, entry: entry} + if p.top.Len() >= p.limit { + p.more = true + if compareMultipartCandidate(c, p.top[0]) >= 0 { + return + } + heap.Pop(&p.top) + } + heap.Push(&p.top, c) +} + +func (p *multipartUploadPage) result() *ListMultipartUploadsResult { + input := p.input output := &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, EncodingType: input.EncodingType, KeyMarker: input.KeyMarker, - MaxUploads: input.MaxUploads, + MaxUploads: aws.Int64(int64(p.limit)), Prefix: input.Prefix, UploadIdMarker: input.UploadIdMarker, IsTruncated: aws.Bool(false), } - if *input.MaxUploads <= 0 { - return output - } - type candidate struct { - key string - entry *filer_pb.Entry - } - var candidates []candidate - for _, entry := range entries { - if !isMultipartUploadEntry(entry) { - continue - } - key := *objectKey(aws.String(string(entry.Extended[s3_constants.ExtMultipartObjectKey]))) - if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) { - continue - } - if *input.KeyMarker != "" && (key < *input.KeyMarker || key == *input.KeyMarker && (*input.UploadIdMarker == "" || entry.Name <= *input.UploadIdMarker)) { - continue - } - candidates = append(candidates, candidate{key: key, entry: entry}) - } - slices.SortFunc(candidates, func(a, b candidate) int { - return cmp.Or(strings.Compare(a.key, b.key), strings.Compare(a.entry.Name, b.entry.Name)) - }) - + candidates := make([]multipartCandidate, len(p.top)) + copy(candidates, p.top) + slices.SortFunc(candidates, compareMultipartCandidate) for _, c := range candidates { - if int64(len(output.Upload)) >= *input.MaxUploads { - last := output.Upload[len(output.Upload)-1] - output.IsTruncated = aws.Bool(true) - output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId - return output - } upload := &s3.MultipartUpload{ Key: aws.String(c.key), UploadId: aws.String(c.entry.Name), @@ -1362,6 +1420,11 @@ func selectMultipartUploads(input *s3.ListMultipartUploadsInput, entries []*file } output.Upload = append(output.Upload, upload) } + if p.more { + last := output.Upload[len(output.Upload)-1] + output.IsTruncated = aws.Bool(true) + output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId + } return output } diff --git a/weed/s3api/filer_multipart_list_test.go b/weed/s3api/filer_multipart_list_test.go index 0ed258fd71..c00f38acaf 100644 --- a/weed/s3api/filer_multipart_list_test.go +++ b/weed/s3api/filer_multipart_list_test.go @@ -1,6 +1,9 @@ package s3api import ( + "context" + "fmt" + "math" "slices" "testing" @@ -8,6 +11,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // uploadEntry is an upload folder as createMultipartUpload leaves it. @@ -112,3 +117,84 @@ func TestSelectMultipartUploadsPaginatesEveryUploadOnce(t *testing.T) { t.Fatalf("listed %d of %d uploads", len(seen), len(entries)) } } + +func TestSelectMultipartUploadsHoldsOnlyThePage(t *testing.T) { + // a folder far larger than the page must not accumulate in the collector: + // peak memory tracks maxUploads, not the open-upload count + page := newMultipartUploadPage(listUploadsInput("", "", "", 3)) + const total = 20000 + for i := 0; i < total; i++ { + page.add(uploadEntry(fmt.Sprintf("%05d", i), fmt.Sprintf("k%05d", total-1-i), int64(i))) + } + if page.top.Len() != 3 { + t.Fatalf("collector holds %d candidates, want the 3-entry page", page.top.Len()) + } + out := page.result() + var got []string + for _, u := range out.Upload { + got = append(got, *u.Key+"/"+*u.UploadId) + } + want := []string{"k00000/19999", "k00001/19998", "k00002/19997"} + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + if !*out.IsTruncated || *out.NextKeyMarker != "k00002" || *out.NextUploadIdMarker != "19997" { + t.Fatalf("truncation markers got (%v, %v, %v)", *out.IsTruncated, *out.NextKeyMarker, *out.NextUploadIdMarker) + } +} + +func TestSelectMultipartUploadsCapsRequestedPageSize(t *testing.T) { + // max-uploads is caller-controlled; the page stays bounded at the server cap + page := newMultipartUploadPage(listUploadsInput("", "", "", 10*maxUploadsList)) + if page.limit != maxUploadsList { + t.Fatalf("page limit %d, want capped at %d", page.limit, maxUploadsList) + } + if got := *page.result().MaxUploads; got != int64(maxUploadsList) { + t.Fatalf("MaxUploads reports %d, want the effective cap %d", got, maxUploadsList) + } +} + +// A transient failure mid-stream replays through a fresh page, so the uploads +// the aborted attempt already handed to the collector must not bleed into the +// result: each upload lists once and the markers track the retried attempt. +func TestListMultipartUploadsRetryCollectsEachUploadOnce(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "c", 10), + uploadEntry("2", "a", 30), + uploadEntry("3", "b", 10), + } + client := &listRetryClient{ + entries: entries, + recvErrs: []error{status.Error(codes.Unavailable, "transport is closing"), nil}, + recvAfter: []int{2, 0}, + } + accessor := &listRetryAccessor{client: client} + input := listUploadsInput("", "", "", 2) + + var page *multipartUploadPage + err := listRetry("/buckets/b/.uploads", func() error { + page = newMultipartUploadPage(input) + return filer_pb.List(context.Background(), accessor, "/buckets/b/.uploads", "", func(entry *filer_pb.Entry, isLastEntry bool) error { + page.add(entry) + return nil + }, "", false, math.MaxInt32) + }) + if err != nil { + t.Fatalf("listRetry: %v", err) + } + if client.attempts != 2 { + t.Fatalf("got %d attempts, want 2", client.attempts) + } + out := page.result() + var got []string + for _, u := range out.Upload { + got = append(got, *u.Key+"/"+*u.UploadId) + } + want := []string{"a/2", "b/3"} + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v (no entry from the aborted attempt may carry over)", got, want) + } + if !*out.IsTruncated || *out.NextKeyMarker != "b" || *out.NextUploadIdMarker != "3" { + t.Fatalf("truncation markers got (%v, %v, %v)", *out.IsTruncated, *out.NextKeyMarker, *out.NextUploadIdMarker) + } +} diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index 5ebffe8f83..3ec242a4d3 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -93,17 +93,17 @@ func isRetryableListError(err error) bool { return err != nil && !isFilerNotFound(err) && util.IsTransientError(err) } -// listWithRetry replays doList while the filer answers with a transient error. +// listRetry replays doList while the filer answers with a transient error. // Both failure points, the ListEntries call itself and the stream.Recv that // follows it, surface as a plain error out of filer_pb.List, so a single retry -// point above it covers both. -func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_pb.Entry, isLast bool, err error)) (entries []*filer_pb.Entry, isLast bool, err error) { +// point above it covers both. A replayed attempt may re-deliver entries the +// last attempt already saw, so a doList that accumulates must reset first. +func listRetry(parentDirectoryPath string, doList func() error) (err error) { backoff := listRetryInitialBackoff for attempt := 1; ; attempt++ { - entries, isLast, err = doList() - if err == nil || attempt >= listRetryAttempts || !isRetryableListError(err) { - return entries, isLast, err + if err = doList(); err == nil || attempt >= listRetryAttempts || !isRetryableListError(err) { + return err } glog.V(1).Infof("list %s attempt %d/%d hit a transient error, retrying in %v: %v", parentDirectoryPath, attempt, listRetryAttempts, backoff, err) time.Sleep(backoff) @@ -112,6 +112,16 @@ func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_p } +func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_pb.Entry, isLast bool, err error)) (entries []*filer_pb.Entry, isLast bool, err error) { + + err = listRetry(parentDirectoryPath, func() error { + entries, isLast, err = doList() + return err + }) + return entries, isLast, err + +} + // A delete is idempotent at the filer, which answers an entry that is already // gone with an empty resp.Error, so a reply the transport dropped can be // reissued instead of surfaced: as a 500 on the bucket delete, or as a per-key diff --git a/weed/s3api/iceberg/commit_helpers.go b/weed/s3api/iceberg/commit_helpers.go index 4caaaeb1a6..116ed85862 100644 --- a/weed/s3api/iceberg/commit_helpers.go +++ b/weed/s3api/iceberg/commit_helpers.go @@ -252,3 +252,59 @@ func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommi Metadata: newMetadata, }, nil } + +// authorizeCommit asks the S3 Tables layer, before anything is written, whether +// the caller may update the table. Both commit paths stage the new metadata +// file in the table's directory before UpdateTable authorizes the pointer +// flip, so without this gate a caller who may only read the table could still +// make the server write and delete files under it. UpdateTable decides again +// when the pointer flips. +func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespace []string, tableName, identityName string) *icebergRequestError { + err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return s.tablesManager.AuthorizeUpdateTable(ctx, s3tables.NewManagerClient(client), &s3tables.UpdateTableRequest{ + TableBucketARN: bucketARN, + Namespace: namespace, + Name: tableName, + }, identityName) + }) + if err == nil { + return nil + } + var tableErr *s3tables.S3TablesError + if errors.As(err, &tableErr) { + switch tableErr.Type { + case s3tables.ErrCodeAccessDenied: + return &icebergRequestError{http.StatusForbidden, "ForbiddenException", fmt.Sprintf("Not authorized to commit to table %s", tableName)} + case s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchNamespace: + return &icebergRequestError{http.StatusNotFound, "NoSuchTableException", fmt.Sprintf("Table does not exist: %s", tableName)} + } + } + glog.V(1).Infof("Iceberg: commit authorization for %s failed: %v", tableName, err) + return &icebergRequestError{http.StatusInternalServerError, "InternalServerError", "Failed to authorize commit: " + err.Error()} +} + +// authorizeViewCommit is authorizeCommit for a view, whose update path stages +// its metadata file before UpdateView authorizes the pointer flip. +func (s *Server) authorizeViewCommit(ctx context.Context, bucketARN string, namespace []string, viewName, identityName string) *icebergRequestError { + err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return s.tablesManager.AuthorizeUpdateView(ctx, s3tables.NewManagerClient(client), &s3tables.UpdateViewRequest{ + TableBucketARN: bucketARN, + Namespace: namespace, + Name: viewName, + }, identityName) + }) + if err == nil { + return nil + } + var tableErr *s3tables.S3TablesError + if errors.As(err, &tableErr) { + switch tableErr.Type { + case s3tables.ErrCodeAccessDenied: + return &icebergRequestError{http.StatusForbidden, "ForbiddenException", fmt.Sprintf("Not authorized to commit to view %s", viewName)} + case s3tables.ErrCodeNoSuchView, s3tables.ErrCodeNoSuchNamespace: + return &icebergRequestError{http.StatusNotFound, "NoSuchViewException", fmt.Sprintf("View does not exist: %s", viewName)} + } + } + glog.V(1).Infof("Iceberg: view commit authorization for %s failed: %v", viewName, err) + return &icebergRequestError{http.StatusInternalServerError, "InternalServerError", "Failed to authorize commit: " + err.Error()} +} diff --git a/weed/s3api/iceberg/handlers_commit.go b/weed/s3api/iceberg/handlers_commit.go index c40ee6b9e7..c31a6d5d5d 100644 --- a/weed/s3api/iceberg/handlers_commit.go +++ b/weed/s3api/iceberg/handlers_commit.go @@ -242,6 +242,10 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) { writeManagerError(w, err) return } + if reqErr := s.authorizeCommit(r.Context(), bucketARN, namespace, tableName, identityName); reqErr != nil { + writeError(w, reqErr.status, reqErr.errType, reqErr.message) + return + } location := tableLocationFromMetadataLocation(getResp.MetadataLocation) if location == "" { diff --git a/weed/s3api/iceberg/handlers_oauth.go b/weed/s3api/iceberg/handlers_oauth.go index 910ffc279a..8e6933c461 100644 --- a/weed/s3api/iceberg/handlers_oauth.go +++ b/weed/s3api/iceberg/handlers_oauth.go @@ -324,20 +324,28 @@ func mintIcebergToken(identityName, accessKey, secret string, ttlSeconds int) (s // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { + identityName, identity, _, ok := s.authenticateBearerKey(r) + return identityName, identity, ok +} + +// authenticateBearerKey is authenticateBearer that also returns the access key +// the token was minted from (its verified access_key claim), so the catalog +// can tell apart the holders of several keys on one identity. +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, false + return "", nil, "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, false + return "", nil, "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, false + return "", nil, "", false } if s.credentialValidator == nil { - return "", nil, false + return "", nil, "", false } // Parse the token without verification first to get the access key, @@ -347,18 +355,18 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to parse token: %v", err) - return "", nil, false + return "", nil, "", false } if unverified.AccessKey == "" { - return "", nil, false + return "", nil, "", false } // Look up the credential by access key to get the signing key for verification identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to get credential for access key: %v", err) - return "", nil, false + return "", nil, "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -371,10 +379,12 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) }) if err != nil || !verified.Valid { glog.V(2).Infof("Iceberg OAuth: token verification failed: %v", err) - return "", nil, false + return "", nil, "", false } - return identityName, identity, true + // The token's signing key is derived from this access key, so a verified + // token was minted from it. + return identityName, identity, unverified.AccessKey, true } // deriveSigningKey derives a signing key from the access key and secret using HMAC-SHA256. diff --git a/weed/s3api/iceberg/handlers_transaction.go b/weed/s3api/iceberg/handlers_transaction.go index 5eed534a87..6279cc4cc7 100644 --- a/weed/s3api/iceberg/handlers_transaction.go +++ b/weed/s3api/iceberg/handlers_transaction.go @@ -140,6 +140,9 @@ func (s *Server) prepareTableCommit(ctx context.Context, bucketName, bucketARN, glog.V(1).Infof("Iceberg: CommitTransaction GetTable error: %v", err) return nil, &icebergRequestError{http.StatusInternalServerError, "InternalServerError", err.Error()} } + if reqErr := s.authorizeCommit(ctx, bucketARN, namespace, tableName, identityName); reqErr != nil { + return nil, reqErr + } location := tableLocationFromMetadataLocation(getResp.MetadataLocation) if location == "" { diff --git a/weed/s3api/iceberg/handlers_view_update.go b/weed/s3api/iceberg/handlers_view_update.go index e7edc7db2f..60971a05fb 100644 --- a/weed/s3api/iceberg/handlers_view_update.go +++ b/weed/s3api/iceberg/handlers_view_update.go @@ -48,6 +48,10 @@ func (s *Server) handleUpdateView(w http.ResponseWriter, r *http.Request) { writeManagerError(w, err) return } + if reqErr := s.authorizeViewCommit(r.Context(), bucketARN, namespace, viewName, identityName); reqErr != nil { + writeError(w, reqErr.status, reqErr.errType, reqErr.message) + return + } if getResp.Metadata == nil || len(getResp.Metadata.FullMetadata) == 0 { writeError(w, http.StatusInternalServerError, "InternalServerError", "view has no metadata") return diff --git a/weed/s3api/iceberg/iceberg_commit_authorize_test.go b/weed/s3api/iceberg/iceberg_commit_authorize_test.go new file mode 100644 index 0000000000..fc13c64dc9 --- /dev/null +++ b/weed/s3api/iceberg/iceberg_commit_authorize_test.go @@ -0,0 +1,237 @@ +package iceberg + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path" + "strings" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/apache/iceberg-go/view" + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "google.golang.org/grpc" +) + +// writeLogFiler records every file the handlers create, including files they +// delete again before answering, which the final entry map cannot show. +type writeLogFiler struct { + *memFiler + created []string +} + +func (f *writeLogFiler) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error { + return fn(f) +} + +func (f *writeLogFiler) CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) { + if !in.Entry.IsDirectory { + f.created = append(f.created, path.Join(in.Directory, in.Entry.Name)) + } + return f.memFiler.CreateEntry(ctx, in, opts...) +} + +// seedOwnedTable registers an existing table owned by owner. +func seedOwnedTable(fc *memFiler, bucket, namespace, tableName, owner string) { + internal := map[string]any{ + "name": tableName, + "namespace": namespace, + "format": "ICEBERG", + "ownerAccountId": owner, + "versionToken": "v1", + "metadataVersion": 1, + "metadataLocation": "s3://" + bucket + "/" + namespace + "/" + tableName + "/metadata/v1.metadata.json", + "metadata": s3tables.TableMetadata{ + Iceberg: &s3tables.IcebergMetadata{TableUUID: "00000000-0000-0000-0000-000000000001"}, + }, + } + metaBytes, _ := json.Marshal(internal) + fc.seed(s3tables.GetTablePath(bucket, namespace, tableName), &filer_pb.Entry{ + Name: tableName, + IsDirectory: true, + Extended: map[string][]byte{s3tables.ExtendedKeyMetadata: metaBytes}, + }) +} + +// seedOwnedView registers an existing view owned by owner. +func seedOwnedView(fc *memFiler, bucket, namespace, viewName, owner string, fullMetadata []byte) { + internal := map[string]any{ + "name": viewName, + "namespace": namespace, + "format": "ICEBERG", + "ownerAccountId": owner, + "versionToken": "v1", + "metadataVersion": 1, + "metadataLocation": "s3://" + bucket + "/" + namespace + "/" + viewName + "/metadata/v1.metadata.json", + } + if len(fullMetadata) > 0 { + internal["metadata"] = s3tables.TableMetadata{FullMetadata: fullMetadata} + } + metaBytes, _ := json.Marshal(internal) + fc.seed(s3tables.GetTablePath(bucket, namespace, viewName), &filer_pb.Entry{ + Name: viewName, + IsDirectory: true, + Extended: map[string][]byte{ + s3tables.ExtendedKeyMetadata: metaBytes, + s3tables.ExtendedKeyEntryType: []byte(s3tables.EntryTypeView), + }, + }) +} + +// withActions gives the request an identity allowed only the listed actions. +func withActions(r *http.Request, name string, actions ...string) *http.Request { + ctx := s3_constants.SetIdentityNameInContext(r.Context(), name) + return r.WithContext(s3_constants.SetIdentityInContext(ctx, &struct { + Name string + Actions []string + }{Name: name, Actions: actions})) +} + +func viewMetadataJSON(t *testing.T, location string) []byte { + t.Helper() + schema := iceberg.NewSchemaWithIdentifiers(0, nil, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true}) + ver, err := view.NewVersionFromSQL(1, schema.ID, "SELECT 1", table.Identifier{"finance"}) + if err != nil { + t.Fatal(err) + } + md, err := view.NewMetadata(ver, schema, location, iceberg.Properties{}) + if err != nil { + t.Fatal(err) + } + fullMetadata, err := json.Marshal(md) + if err != nil { + t.Fatal(err) + } + return fullMetadata +} + +// A caller that may load a table but not commit to it must be refused before +// the commit stages its metadata file in the table's directory. The commit +// paths used to write that file, reach the UpdateTable authorization, and +// delete the file again on the way out. +func TestCommitDeniedBeforeStagingMetadata(t *testing.T) { + const bucket = "warehouse" + cases := map[string]func(s *Server) *httptest.ResponseRecorder{ + "commit-table": func(s *Server) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/tables/quarterly_reports", + strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`)) + r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "table": "quarterly_reports"}) + w := httptest.NewRecorder() + s.handleUpdateTable(w, withActions(r, "reader", "s3tables:GetTable")) + return w + }, + "commit-transaction": func(s *Server) *httptest.ResponseRecorder { + body := `{"table-changes":[{"identifier":{"namespace":["finance"],"name":"quarterly_reports"},"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}]}` + r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/transactions/commit", strings.NewReader(body)) + r = mux.SetURLVars(r, map[string]string{"prefix": bucket}) + w := httptest.NewRecorder() + s.handleCommitTransaction(w, withActions(r, "reader", "s3tables:GetTable")) + return w + }, + "commit-view": func(s *Server) *httptest.ResponseRecorder { + r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/views/quarterly_view", + strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`)) + r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "view": "quarterly_view"}) + w := httptest.NewRecorder() + s.handleUpdateView(w, withActions(r, "reader", "s3tables:GetView")) + return w + }, + } + for name, run := range cases { + t.Run(name, func(t *testing.T) { + fc := &writeLogFiler{memFiler: newMemFiler()} + seedNamespace(fc.memFiler, bucket, "finance", "alice") + seedOwnedTable(fc.memFiler, bucket, "finance", "quarterly_reports", "alice") + seedOwnedView(fc.memFiler, bucket, "finance", "quarterly_view", "alice", viewMetadataJSON(t, "s3://"+bucket+"/finance/quarterly_view")) + s := NewServer(fc, nil) + + w := run(s) + + if w.Code != http.StatusForbidden { + t.Errorf("status = %d, want %d (body: %s)", w.Code, http.StatusForbidden, w.Body.String()) + } + if len(fc.created) != 0 { + t.Errorf("refused commit wrote %v", fc.created) + } + }) + } +} + +// The owner, and a caller whose own identity carries UpdateTable, still +// stage their metadata file and land the commit. +func TestCommitAllowedStagesMetadata(t *testing.T) { + const bucket = "warehouse" + callers := map[string]func(r *http.Request) *http.Request{ + "owner": func(r *http.Request) *http.Request { + return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice")) + }, + "identity-actions": func(r *http.Request) *http.Request { + return withActions(r, "writer", "s3tables:GetTable", "s3tables:UpdateTable") + }, + } + for name, as := range callers { + t.Run(name, func(t *testing.T) { + fc := &writeLogFiler{memFiler: newMemFiler()} + seedNamespace(fc.memFiler, bucket, "finance", "alice") + seedOwnedTable(fc.memFiler, bucket, "finance", "quarterly_reports", "alice") + s := NewServer(fc, nil) + + r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/tables/quarterly_reports", + strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`)) + r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "table": "quarterly_reports"}) + w := httptest.NewRecorder() + s.handleUpdateTable(w, as(r)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusOK, w.Body.String()) + } + if len(fc.created) != 1 || !strings.HasSuffix(fc.created[0], ".metadata.json") { + t.Fatalf("created = %v, want one metadata file", fc.created) + } + }) + } +} + +// Same staging gate on the view commit path: the owner, and a caller whose own +// identity carries UpdateView, stage a metadata file and land the commit. +func TestViewCommitAllowedStagesMetadata(t *testing.T) { + const bucket = "warehouse" + fullMetadata := viewMetadataJSON(t, "s3://"+bucket+"/finance/quarterly_view") + + callers := map[string]func(r *http.Request) *http.Request{ + "owner": func(r *http.Request) *http.Request { + return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), "alice")) + }, + "identity-actions": func(r *http.Request) *http.Request { + return withActions(r, "writer", "s3tables:GetView", "s3tables:UpdateView") + }, + } + for name, as := range callers { + t.Run(name, func(t *testing.T) { + fc := &writeLogFiler{memFiler: newMemFiler()} + seedNamespace(fc.memFiler, bucket, "finance", "alice") + seedOwnedView(fc.memFiler, bucket, "finance", "quarterly_view", "alice", fullMetadata) + s := NewServer(fc, nil) + + r := httptest.NewRequest(http.MethodPost, "/v1/"+bucket+"/namespaces/finance/views/quarterly_view", + strings.NewReader(`{"requirements":[],"updates":[{"action":"set-properties","updates":{"k":"v"}}]}`)) + r = mux.SetURLVars(r, map[string]string{"prefix": bucket, "namespace": "finance", "view": "quarterly_view"}) + w := httptest.NewRecorder() + s.handleUpdateView(w, as(r)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d (body: %s)", w.Code, http.StatusOK, w.Body.String()) + } + if len(fc.created) != 1 || !strings.HasSuffix(fc.created[0], ".metadata.json") { + t.Fatalf("created = %v, want one metadata file", fc.created) + } + }) + } +} diff --git a/weed/s3api/iceberg/server.go b/weed/s3api/iceberg/server.go index 5403654753..5ec8de2160 100644 --- a/weed/s3api/iceberg/server.go +++ b/weed/s3api/iceberg/server.go @@ -269,9 +269,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // The auth scheme is case-insensitive (RFC 7235), matching // authenticateBearer below. if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, ok := s.authenticateBearer(r); ok { + if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) + ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } @@ -290,6 +291,9 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return } + // The verifier records the key of the credential that matched into + // this holder; the vended credentials name it. + r = s3_constants.EnsureIdentityHolder(r) identityName, identity, errCode := s.authenticator.AuthenticateRequest(r) if errCode != s3err.ErrNone { apiErr := s3err.GetAPIError(errCode) diff --git a/weed/s3api/iceberg/server_access_key_test.go b/weed/s3api/iceberg/server_access_key_test.go new file mode 100644 index 0000000000..80ef9923b5 --- /dev/null +++ b/weed/s3api/iceberg/server_access_key_test.go @@ -0,0 +1,97 @@ +package iceberg + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// keyedS3Authenticator accepts every SigV4 request as one identity and records +// the key that verified into the request's holder, as S3ApiServer's verifiers do. +type keyedS3Authenticator struct { + mockS3Authenticator + accessKey string +} + +func (k *keyedS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { + s3_constants.RecordAccessKeyId(r.Context(), k.accessKey) + return "shared-identity", "identity-object", s3err.ErrNone +} + +// A catalog token is minted from one access key; the request it authenticates +// carries that key, so the credentials it vends can name it. +func TestAuthBearerRecordsTheTokensAccessKey(t *testing.T) { + s, _ := newAuthTestServer() + now := time.Now() + token := mintTestToken(t, "AKID123", "secret456", now, now.Add(time.Hour)) + + var gotKey, gotName string + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotKey = s3_constants.GetAccessKeyIdFromContext(r) + gotName = s3_constants.GetIdentityNameFromContext(r) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + handler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("valid Bearer: status = %d, want 200", rec.Code) + } + if gotName != "testuser" { + t.Errorf("identity = %q, want testuser", gotName) + } + if gotKey != "AKID123" { + t.Errorf("access key in context = %q, want AKID123", gotKey) + } +} + +// A SigV4-signed catalog request carries the key its authenticator verified. +func TestAuthSigV4RecordsTheAuthenticatorsAccessKey(t *testing.T) { + s := newTestServerWithOAuth() + s.authenticator = &keyedS3Authenticator{accessKey: "AKIDSIGV4"} + + var gotKey string + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotKey = s3_constants.GetAccessKeyIdFromContext(r) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIDSIGV4/20261016/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=00") + rec := httptest.NewRecorder() + handler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if gotKey != "AKIDSIGV4" { + t.Errorf("access key in context = %q, want AKIDSIGV4", gotKey) + } +} + +// An authenticator that records no key leaves the context without one. +func TestAuthSigV4WithoutKeyAuthenticatorRecordsNoKey(t *testing.T) { + s := newTestServerWithOAuth() + s.authenticator = &mockS3Authenticator{} + + gotKey := "unset" + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotKey = s3_constants.GetAccessKeyIdFromContext(r) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/v1/namespaces", nil) + rec := httptest.NewRecorder() + handler(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if gotKey != "" { + t.Errorf("access key in context = %q, want none", gotKey) + } +} diff --git a/weed/s3api/lance/handlers_oauth.go b/weed/s3api/lance/handlers_oauth.go index 5e7003a743..20e791221e 100644 --- a/weed/s3api/lance/handlers_oauth.go +++ b/weed/s3api/lance/handlers_oauth.go @@ -147,20 +147,27 @@ func mintToken(identityName, accessKey, secret string, ttlSeconds int) (string, // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { + identityName, identity, _, ok := s.authenticateBearerKey(r) + return identityName, identity, ok +} + +// authenticateBearerKey is authenticateBearer that also returns the access key +// the token was minted from (its verified access_key claim). +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, false + return "", nil, "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, false + return "", nil, "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, false + return "", nil, "", false } if s.credentialValidator == nil { - return "", nil, false + return "", nil, "", false } // Parse the token without verification first to get the access key, @@ -170,17 +177,17 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Lance OAuth: failed to parse token: %v", err) - return "", nil, false + return "", nil, "", false } if unverified.AccessKey == "" { - return "", nil, false + return "", nil, "", false } identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Lance OAuth: failed to get credential for access key: %v", err) - return "", nil, false + return "", nil, "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -193,10 +200,12 @@ func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) }) if err != nil || !verified.Valid { glog.V(2).Infof("Lance OAuth: token verification failed: %v", err) - return "", nil, false + return "", nil, "", false } - return identityName, identity, true + // The token's signing key is derived from this access key, so a verified + // token was minted from it. + return identityName, identity, unverified.AccessKey, true } // authenticateApiKey validates an x-api-key header carrying an S3 credential diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index 936bd391e7..d0cb813daa 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -177,9 +177,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // different error that clients cannot act on. // The auth scheme is case-insensitive (RFC 7235). if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, ok := s.authenticateBearer(r); ok { + if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) + ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } @@ -194,8 +195,10 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { if apiKey := r.Header.Get("x-api-key"); apiKey != "" { if identityName, identity, ok := s.authenticateApiKey(apiKey); ok { + accessKey, _, _ := strings.Cut(apiKey, ":") ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) + ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } @@ -212,6 +215,9 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return } + // The verifier records the key of the credential that matched into + // this holder; the vended credentials name it. + r = s3_constants.EnsureIdentityHolder(r) identityName, identity, errCode := s.authenticator.AuthenticateRequest(r) if errCode != s3err.ErrNone { apiErr := s3err.GetAPIError(errCode) diff --git a/weed/s3api/lance/server_access_key_test.go b/weed/s3api/lance/server_access_key_test.go new file mode 100644 index 0000000000..3013256453 --- /dev/null +++ b/weed/s3api/lance/server_access_key_test.go @@ -0,0 +1,69 @@ +package lance + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// keyedS3Authenticator accepts every request as one identity and records the +// key that verified into the request's holder, as S3ApiServer's verifiers do. +type keyedS3Authenticator struct { + mockS3Authenticator + accessKey string +} + +func (k *keyedS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { + s3_constants.RecordAccessKeyId(r.Context(), k.accessKey) + return "s3user", nil, s3err.ErrNone +} + +func accessKeySeenByHandler(t *testing.T, s *Server, req *http.Request) string { + t.Helper() + gotKey := "unset" + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotKey = s3_constants.GetAccessKeyIdFromContext(r) + w.WriteHeader(http.StatusOK) + }) + rec := httptest.NewRecorder() + handler(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + return gotKey +} + +// Each way a Lance client authenticates leaves the access key it used in the +// request context, so the credentials it vends can name that key. +func TestAuthRecordsTheCallersAccessKey(t *testing.T) { + now := time.Now() + + s := newTestServerWithOAuth() + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + req.Header.Set("Authorization", "Bearer "+mintTestToken(t, "AKID123", "secret456", now, now.Add(time.Hour))) + if got := accessKeySeenByHandler(t, s, req); got != "AKID123" { + t.Errorf("Bearer: access key in context = %q, want AKID123", got) + } + + req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + req.Header.Set("x-api-key", "AKID123:secret456") + if got := accessKeySeenByHandler(t, s, req); got != "AKID123" { + t.Errorf("x-api-key: access key in context = %q, want AKID123", got) + } + + s.authenticator = &keyedS3Authenticator{accessKey: "AKIDSIGV4"} + req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + if got := accessKeySeenByHandler(t, s, req); got != "AKIDSIGV4" { + t.Errorf("SigV4: access key in context = %q, want AKIDSIGV4", got) + } + + s.authenticator = &mockS3Authenticator{} + req = httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + if got := accessKeySeenByHandler(t, s, req); got != "" { + t.Errorf("authenticator without a key: access key in context = %q, want none", got) + } +} diff --git a/weed/s3api/policy_engine/README_POLICY_ENGINE.md b/weed/s3api/policy_engine/README_POLICY_ENGINE.md index b4846628da..97c4344e4a 100644 --- a/weed/s3api/policy_engine/README_POLICY_ENGINE.md +++ b/weed/s3api/policy_engine/README_POLICY_ENGINE.md @@ -356,6 +356,7 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | `aws:SecureTransport` | Require HTTPS | | `aws:CurrentTime` | Time-based access control | | `s3:ExistingObjectTag/` | Tag-based access control for existing objects | +| `s3:RequestObjectTag/` | Tag conditions on the tags a PUT request sets | | Wildcard Patterns | Support for `*` and `?` in actions and resources | | Principal Matching | `*`, account IDs, and user ARNs | @@ -363,7 +364,6 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | Feature | GitHub Issue | |---------|--------------| -| `s3:RequestObjectTag/` | For tag conditions on PUT requests | | `s3:RequestObjectTagKeys` | Check which tag keys are in request | | `s3:x-amz-content-sha256` | Content hash condition | | `s3:x-amz-server-side-encryption` | SSE condition | diff --git a/weed/s3api/policy_engine/conditions.go b/weed/s3api/policy_engine/conditions.go index becf8836da..a995c5ddb9 100644 --- a/weed/s3api/policy_engine/conditions.go +++ b/weed/s3api/policy_engine/conditions.go @@ -728,6 +728,10 @@ func GetConditionEvaluator(operator string) (ConditionEvaluator, error) { // ExistingObjectTagPrefix is the prefix for S3 policy condition keys const ExistingObjectTagPrefix = "s3:ExistingObjectTag/" +// RequestObjectTagPrefix is the prefix for condition keys on the tags a +// request sets through the X-Amz-Tagging header. +const RequestObjectTagPrefix = "s3:RequestObjectTag/" + // getConditionContextValue resolves the value(s) for a condition key. // For s3:ExistingObjectTag/ conditions, it looks up the tag in objectEntry. // For other condition keys, it looks up the value in contextValues. diff --git a/weed/s3api/policy_engine/engine.go b/weed/s3api/policy_engine/engine.go index 933ed1d96c..0275f33627 100644 --- a/weed/s3api/policy_engine/engine.go +++ b/weed/s3api/policy_engine/engine.go @@ -3,6 +3,7 @@ package policy_engine import ( "fmt" "net/http" + "net/url" "regexp" "strings" "sync" @@ -10,6 +11,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" ) // PolicyEvaluationResult represents the result of policy evaluation @@ -439,6 +441,10 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m // Note: s3:ExistingObjectTag/ conditions are evaluated using objectEntry // passed to EvaluatePolicy, not extracted from the request. + for key, value := range requestObjectTags(r) { + values[RequestObjectTagPrefix+key] = []string{value} + } + // S3 bucket-level conditions if delimiter := r.URL.Query().Get("delimiter"); delimiter != "" { values["s3:delimiter"] = []string{delimiter} @@ -496,6 +502,86 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m return values } +// requestObjectTags returns the tags that X-Amz-Tagging will store on the +// object, or nil when the handler ignores the header — otherwise a tag +// attached to a read or to a write that never stores it would satisfy a +// tag-conditioned policy the object cannot meet. +func requestObjectTags(r *http.Request) map[string]string { + tagging := r.Header.Get(s3_constants.AmzObjectTagging) + if tagging == "" { + return nil + } + if !storesRequestObjectTags(r) { + return nil + } + // A malformed header is rejected by the handler with InvalidTag. + parsed, err := url.ParseQuery(tagging) + if err != nil { + return nil + } + // putToFiler rejects a repeated key, but CopyObject stores its last + // value, so that is the one the policy has to see. + tags := make(map[string]string, len(parsed)) + for key, vals := range parsed { + tags[key] = vals[len(vals)-1] + } + return tags +} + +// storesRequestObjectTags reports whether the request's handler saves the +// X-Amz-Tagging header on the object: PutObject, a copy under +// X-Amz-Tagging-Directive: REPLACE, and CreateMultipartUpload do; every +// other operation — reads, deletes, part uploads, completion, tagging +// subresources — does not. +func storesRequestObjectTags(r *http.Request) bool { + query := r.URL.Query() + if r.Method == http.MethodPost { + return query.Has("uploads") && !query.Has("uploadId") + } + if r.Method != http.MethodPut { + return false + } + // A numeric partNumber together with uploadId routes to the part + // handlers — with or without a copy source — which never read + // X-Amz-Tagging. + if query.Has("partNumber") && query.Has("uploadId") && isDigits(query.Get("partNumber")) { + return false + } + // The PUT subresources route before CopyObject and never read + // X-Amz-Tagging; any other query key — presign parameters included — + // still reaches the object PUT handlers. + for _, subresource := range []string{"tagging", "renameObject", "acl", "retention", "legal-hold"} { + if query.Has(subresource) { + return false + } + } + if r.Header.Get("X-Amz-Copy-Source") != "" { + return r.Header.Get(s3_constants.AmzObjectTaggingDirective) == "REPLACE" + } + if query.Has("partNumber") || query.Has("uploadId") { + return false + } + // An explicit directory-marker PUT stores no tags. + if strings.HasSuffix(r.URL.Path, "/") && r.ContentLength <= 1024 { + return false + } + return true +} + +// isDigits reports whether s is non-empty and all ASCII digits, matching the +// partNumber route constraint. +func isDigits(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return true +} + // IsMultipartContinuationAction returns true for actions that do not carry // their own SSE header because SSE is inherited from CreateMultipartUpload. func IsMultipartContinuationAction(action string) bool { diff --git a/weed/s3api/policy_engine/engine_test.go b/weed/s3api/policy_engine/engine_test.go index d527e54bb3..587ffdc115 100644 --- a/weed/s3api/policy_engine/engine_test.go +++ b/weed/s3api/policy_engine/engine_test.go @@ -896,6 +896,161 @@ func TestExistingObjectTagCondition(t *testing.T) { } } +func TestRequestObjectTagCondition(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "StringEquals": { + "s3:RequestObjectTag/department": ["engineering"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + copyHeaders := map[string]string{"X-Amz-Copy-Source": "/src-bucket/src-object"} + replaceHeaders := map[string]string{ + "X-Amz-Copy-Source": "/src-bucket/src-object", + s3_constants.AmzObjectTaggingDirective: "REPLACE", + } + + tests := []struct { + name string + method string + target string + headers map[string]string + tagging string + expected PolicyEvaluationResult + }{ + {"matching tag allows", "", "", nil, "department=engineering", PolicyResultAllow}, + {"matching tag among others allows", "", "", nil, "project=alpha&department=engineering", PolicyResultAllow}, + {"url-encoded value is decoded", "", "", nil, "department=engin%65ering", PolicyResultAllow}, + {"other value is not allowed", "", "", nil, "department=sales", PolicyResultIndeterminate}, + {"missing tag is not allowed", "", "", nil, "project=alpha", PolicyResultIndeterminate}, + {"no tagging header is not allowed", "", "", nil, "", PolicyResultIndeterminate}, + {"malformed header is not allowed", "", "", nil, "department=engineering&%zz", PolicyResultIndeterminate}, + {"last value of a repeated key counts", "", "", nil, "department=sales&department=engineering", PolicyResultAllow}, + {"earlier value of a repeated key is ignored", "", "", nil, "department=engineering&department=sales", PolicyResultIndeterminate}, + {"header is ignored on PutObjectTagging", "", "?tagging", nil, "department=engineering", PolicyResultIndeterminate}, + {"header counts with an unrelated query parameter", "", "?foo=bar", nil, "department=engineering", PolicyResultAllow}, + {"header counts on a Signature V2 presigned PUT", "", "?AWSAccessKeyId=k&Signature=s&Expires=1", nil, "department=engineering", PolicyResultAllow}, + {"header is ignored on a renameObject subresource", "", "?renameObject", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a copy that keeps source tags", "", "", copyHeaders, "department=engineering", PolicyResultIndeterminate}, + {"header counts on a copy that replaces tags", "", "", replaceHeaders, "department=engineering", PolicyResultAllow}, + {"header is ignored on a multipart copy that replaces tags", "", "?partNumber=1&uploadId=x", replaceHeaders, "department=engineering", PolicyResultIndeterminate}, + {"header counts on a copy with a lone partNumber", "", "?partNumber=1", replaceHeaders, "department=engineering", PolicyResultAllow}, + {"header counts on a multipart create", "POST", "?uploads", nil, "department=engineering", PolicyResultAllow}, + {"header is ignored on a part upload", "", "?partNumber=1&uploadId=x", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on multipart completion", "POST", "?uploadId=x", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a directory marker", "", "/", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on an acl subresource", "", "?acl", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a GET", "GET", "", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a DELETE", "DELETE", "", nil, "department=engineering", PolicyResultIndeterminate}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + method := tt.method + if method == "" { + method = http.MethodPut + } + req := httptest.NewRequest(method, "/test-bucket/test-object"+tt.target, nil) + for k, v := range tt.headers { + req.Header.Set(k, v) + } + if tt.tagging != "" { + req.Header.Set(s3_constants.AmzObjectTagging, tt.tagging) + } + + args := &PolicyEvaluationArgs{ + Action: "s3:PutObject", + Resource: "arn:aws:s3:::test-bucket/test-object", + Principal: "*", + Conditions: engine.ExtractConditionValuesFromRequest(req), + } + + if result := engine.EvaluatePolicy("test-bucket", args); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestRequestObjectTagConditionDeny(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*" + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "StringNotEquals": { + "s3:RequestObjectTag/department": ["engineering"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + tests := []struct { + name string + target string + tagging string + expected PolicyEvaluationResult + }{ + {"required tag passes", "", "department=engineering", PolicyResultAllow}, + {"other value is denied", "", "department=sales", PolicyResultDeny}, + {"missing tag is denied", "", "project=alpha", PolicyResultDeny}, + {"repeated key ending in another value is denied", "", "department=engineering&department=sales", PolicyResultDeny}, + {"malformed header is denied", "", "department=engineering&%zz", PolicyResultDeny}, + {"unrelated query parameter does not hide the tag", "?foo=bar", "department=sales", PolicyResultDeny}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/test-bucket/test-object"+tt.target, nil) + req.Header.Set(s3_constants.AmzObjectTagging, tt.tagging) + + args := &PolicyEvaluationArgs{ + Action: "s3:PutObject", + Resource: "arn:aws:s3:::test-bucket/test-object", + Principal: "*", + Conditions: engine.ExtractConditionValuesFromRequest(req), + } + + if result := engine.EvaluatePolicy("test-bucket", args); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + // TestExistingObjectTagConditionMultipleTags tests policies with multiple tag conditions func TestExistingObjectTagConditionMultipleTags(t *testing.T) { engine := NewPolicyEngine() diff --git a/weed/s3api/s3_action_resolver.go b/weed/s3api/s3_action_resolver.go index 7c228ba2ab..ea33de78d0 100644 --- a/weed/s3api/s3_action_resolver.go +++ b/weed/s3api/s3_action_resolver.go @@ -166,15 +166,25 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool) } } - // ACL operations + // ACL operations. A versionId retargets the request at a specific + // version, so the version ACL action applies, not the current-object one. if query.Has("acl") { + versioned := hasObject && query.Get("versionId") != "" switch method { - case http.MethodGet, http.MethodHead: + // HEAD has no acl route; it lands on HeadObjectHandler and should + // resolve like a read — s3:GetObject, or s3:GetObjectVersion below. + case http.MethodGet: + if versioned { + return s3_constants.S3_ACTION_GET_OBJECT_VERSION_ACL + } if hasObject { return s3_constants.S3_ACTION_GET_OBJECT_ACL } return s3_constants.S3_ACTION_GET_BUCKET_ACL case http.MethodPut: + if versioned { + return s3_constants.S3_ACTION_PUT_OBJECT_VERSION_ACL + } if hasObject { return s3_constants.S3_ACTION_PUT_OBJECT_ACL } @@ -220,6 +230,9 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool) // Must be checked before versionId, because GET /bucket/key?attributes&versionId=xyz // is a GetObjectAttributes request, not a GetObjectVersion request if hasObject && query.Has("attributes") && method == http.MethodGet { + if query.Get("versionId") != "" { + return s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES + } return s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES } diff --git a/weed/s3api/s3_action_resolver_test.go b/weed/s3api/s3_action_resolver_test.go index e46c1c8b6a..d073355fd5 100644 --- a/weed/s3api/s3_action_resolver_test.go +++ b/weed/s3api/s3_action_resolver_test.go @@ -83,7 +83,7 @@ func TestResolveS3Action_AttributesBeforeVersionId(t *testing.T) { method: http.MethodGet, baseAction: s3_constants.ACTION_READ, object: "key", - want: s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES, + want: s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES, }, { name: "versionId only GET", @@ -146,6 +146,59 @@ func TestResolveS3Action_VersionTagging(t *testing.T) { } } +// An ACL request carrying versionId targets a specific version and must +// resolve to the version ACL action so a grant limited to the current-object +// action cannot reach historical versions. +func TestResolveS3Action_VersionAcl(t *testing.T) { + tests := []struct { + name string + method string + object string + query string + want string + }{ + {"get current acl", http.MethodGet, "key", "acl", s3_constants.S3_ACTION_GET_OBJECT_ACL}, + {"put current acl", http.MethodPut, "key", "acl", s3_constants.S3_ACTION_PUT_OBJECT_ACL}, + {"get version acl", http.MethodGet, "key", "acl&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_ACL}, + {"head acl resolves as head object", http.MethodHead, "key", "acl", s3_constants.S3_ACTION_GET_OBJECT}, + {"head version acl resolves as head version", http.MethodHead, "key", "acl&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION}, + {"put version acl", http.MethodPut, "key", "acl&versionId=v1", s3_constants.S3_ACTION_PUT_OBJECT_VERSION_ACL}, + {"empty versionId stays current", http.MethodGet, "key", "acl&versionId=", s3_constants.S3_ACTION_GET_OBJECT_ACL}, + {"bucket acl ignores versionId", http.MethodGet, "", "acl&versionId=v1", s3_constants.S3_ACTION_GET_BUCKET_ACL}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil) + if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want { + t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want) + } + }) + } +} + +// An attributes request carrying versionId targets a specific version and +// must resolve to the version attributes action. +func TestResolveS3Action_VersionAttributes(t *testing.T) { + tests := []struct { + name string + method string + object string + query string + want string + }{ + {"get current attributes", http.MethodGet, "key", "attributes", s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES}, + {"get version attributes", http.MethodGet, "key", "attributes&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil) + if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want { + t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want) + } + }) + } +} + // Bucket subresources registered with ACTION_ADMIN must resolve to their own // S3 actions so a policy granting one of them does not need s3:*, and so no // broader grant sweeps them in. diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 28a2d1baf0..cf26ac8f84 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -216,10 +216,21 @@ func GetBucketAndObject(r *http.Request) (bucket, object string) { return } -// IsValidObjectKey rejects S3 object keys that — after normalization -// (backslash→slash, slash collapse) — contain a `.` or `..` path segment, or -// embed a NUL byte. Such keys are collapsed by filepath.Join inside the filer -// and would escape the bucket directory, so they must not reach the gRPC layer. +// allowBackslashInObjectKey keeps '\' a literal object-key character instead +// of normalizing it to '/'. It is a process-wide mode set once at S3 server +// startup; the filer only treats '/' as a path separator, so a verbatim '\' +// stays inside one filer entry name. +var allowBackslashInObjectKey atomic.Bool + +func SetAllowBackslashInObjectKey(allow bool) { + allowBackslashInObjectKey.Store(allow) +} + +// IsValidObjectKey rejects S3 object keys that — after the mode-dependent +// normalization NormalizeObjectKey applies — contain a `.` or `..` path +// segment, or embed a NUL byte. Such keys are collapsed by filepath.Join +// inside the filer and would escape the bucket directory, so they must not +// reach the gRPC layer. // Gorilla mux URL-decodes captured vars before this runs, so `%2e%2e` is // already `..` here. func IsValidObjectKey(object string) bool { @@ -229,7 +240,9 @@ func IsValidObjectKey(object string) bool { if strings.ContainsRune(object, '\x00') { return false } - object = strings.ReplaceAll(object, "\\", "/") + if !allowBackslashInObjectKey.Load() { + object = strings.ReplaceAll(object, "\\", "/") + } for _, seg := range strings.Split(object, "/") { if seg == "." || seg == ".." { return false @@ -263,7 +276,8 @@ func IsValidPathSegment(value string) bool { // NormalizeObjectKey normalizes object keys by removing duplicate slashes and converting backslashes. // This normalizes keys from various sources (URL path, form values, etc.) to a consistent format. -// It also converts Windows-style backslashes to forward slashes for cross-platform compatibility. +// It also converts Windows-style backslashes to forward slashes for cross-platform compatibility, +// unless allowBackslashInObjectKey is set. // Returns keys WITHOUT leading slash to match S3 API format (e.g., "foo/bar" not "/foo/bar"). // Preserves trailing slash if present (e.g., "foo/" stays "foo/"). func NormalizeObjectKey(object string) string { @@ -271,7 +285,9 @@ func NormalizeObjectKey(object string) string { hasTrailingSlash := strings.HasSuffix(object, "/") // Convert Windows-style backslashes to forward slashes - object = strings.ReplaceAll(object, "\\", "/") + if !allowBackslashInObjectKey.Load() { + object = strings.ReplaceAll(object, "\\", "/") + } object = removeDuplicateSlashes(object) // Remove leading slash to match S3 API format object = strings.TrimPrefix(object, "/") @@ -338,6 +354,7 @@ const ( contextKeyIdentityClaim contextKey = "s3-identity-claim" contextKeyIgnoredSessionToken contextKey = "s3-ignored-session-token" contextKeyOriginalRequestPath contextKey = "s3-original-request-path" + contextKeyAccessKeyId contextKey = "s3-access-key-id" ) // identityHolder is a mutable container for the authenticated identity name, @@ -351,6 +368,7 @@ type identityHolder struct { name atomic.Pointer[string] principalArn atomic.Pointer[string] identityClaim atomic.Pointer[string] + accessKeyId atomic.Pointer[string] } // EnsureIdentityHolder attaches a mutable identity holder to the request context @@ -480,6 +498,62 @@ func GetOriginalRequestPathFromContext(r *http.Request) string { return "" } +// SetAccessKeyIdInContext stores the access key id the request authenticated +// with: a static key's id, or an STS session's temporary key id. Several keys +// can belong to one identity, so the identity name alone cannot tell their +// holders apart in the audit log. An access key id is not a secret (AWS logs +// it in CloudTrail); the secret key never reaches the context. +func SetAccessKeyIdInContext(ctx context.Context, accessKeyId string) context.Context { + if accessKeyId == "" { + return ctx + } + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.accessKeyId.Store(&accessKeyId) + } + return context.WithValue(ctx, contextKeyAccessKeyId, accessKeyId) +} + +// RecordAccessKeyId stores the access key id of a credential whose signature +// was just verified into the request's identity holder, so the outer +// middleware (the audit fallback) and the handler both see it. A no-op without +// a holder (EnsureIdentityHolder) or without a key. Only a verifier calls it: +// the key comes from the credential that matched, never from re-reading the +// request, whose shape the verifier already resolved. +func RecordAccessKeyId(ctx context.Context, accessKeyId string) { + if ctx == nil || accessKeyId == "" { + return + } + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.accessKeyId.Store(&accessKeyId) + } +} + +// GetAccessKeyIdFromContext retrieves the access key id the request +// authenticated with, or "" when it carried none (anonymous, OIDC bearer). +func GetAccessKeyIdFromContext(r *http.Request) string { + if r == nil { + return "" + } + return AccessKeyIdFromContext(r.Context()) +} + +// AccessKeyIdFromContext is GetAccessKeyIdFromContext for code that holds +// only the request's context, such as the Iceberg credential vendor. +func AccessKeyIdFromContext(ctx context.Context) string { + if ctx == nil { + return "" + } + if id, ok := ctx.Value(contextKeyAccessKeyId).(string); ok && id != "" { + return id + } + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + if id := h.accessKeyId.Load(); id != nil { + return *id + } + } + return "" +} + // IgnoreSessionTokenInContext marks the request's session token as tolerated // rather than authenticated: a statically configured access key signed the // request, so the token (which credential vendors like Unity Catalog emit on diff --git a/weed/s3api/s3_constants/header_test.go b/weed/s3api/s3_constants/header_test.go index 7e45882710..de780d0d51 100644 --- a/weed/s3api/s3_constants/header_test.go +++ b/weed/s3api/s3_constants/header_test.go @@ -123,6 +123,54 @@ func TestIsValidObjectKey(t *testing.T) { } } +func TestNormalizeObjectKeyAllowBackslash(t *testing.T) { + SetAllowBackslashInObjectKey(true) + defer SetAllowBackslashInObjectKey(false) + + normalizeTests := []struct { + name string + input string + expected string + }{ + {"backslash kept verbatim", "folder\\file.txt", "folder\\file.txt"}, + {"nested backslash kept", "folder\\sub\\file.txt", "folder\\sub\\file.txt"}, + {"trailing backslash kept", "folder\\", "folder\\"}, + {"lone backslash kept", "\\", "\\"}, + {"double backslash kept", "a\\\\b", "a\\\\b"}, + {"mixed separators", "folder\\sub/another\\file.txt", "folder\\sub/another\\file.txt"}, + {"duplicate slashes still collapse", "folder//sub///file.txt", "folder/sub/file.txt"}, + {"leading slash still trimmed", "/folder\\file.txt", "folder\\file.txt"}, + {"trailing slash preserved", "folder/", "folder/"}, + {"windows path style", "C:\\Users\\test\\file.txt", "C:\\Users\\test\\file.txt"}, + } + for _, tt := range normalizeTests { + t.Run(tt.name, func(t *testing.T) { + if got := NormalizeObjectKey(tt.input); got != tt.expected { + t.Errorf("NormalizeObjectKey(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } + + validTests := []struct { + name string + input string + want bool + }{ + {"backslash dotdot is one segment", "..\\evil\\test.txt", true}, + {"backslash dotdot inside", "good\\..\\evil/test.txt", true}, + {"nested dotdot still rejected", "good/../evil/test.txt", false}, + {"leading dotdot still rejected", "../evil/test.txt", false}, + {"plain backslash key", "folder\\file.txt", true}, + } + for _, tt := range validTests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidObjectKey(tt.input); got != tt.want { + t.Errorf("IsValidObjectKey(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} + func TestIsValidBucketName(t *testing.T) { tests := []struct { name string diff --git a/weed/s3api/s3_constants/s3_action_strings.go b/weed/s3api/s3_constants/s3_action_strings.go index 059c6e1dd8..124a29e5d2 100644 --- a/weed/s3api/s3_constants/s3_action_strings.go +++ b/weed/s3api/s3_constants/s3_action_strings.go @@ -4,17 +4,22 @@ package s3_constants // These match the official AWS S3 action format used in IAM and bucket policies const ( // Object operations - S3_ACTION_GET_OBJECT = "s3:GetObject" - S3_ACTION_PUT_OBJECT = "s3:PutObject" - S3_ACTION_DELETE_OBJECT = "s3:DeleteObject" - S3_ACTION_DELETE_OBJECT_VERSION = "s3:DeleteObjectVersion" - S3_ACTION_GET_OBJECT_VERSION = "s3:GetObjectVersion" - S3_ACTION_GET_OBJECT_ATTRIBUTES = "s3:GetObjectAttributes" + S3_ACTION_GET_OBJECT = "s3:GetObject" + S3_ACTION_PUT_OBJECT = "s3:PutObject" + S3_ACTION_DELETE_OBJECT = "s3:DeleteObject" + S3_ACTION_DELETE_OBJECT_VERSION = "s3:DeleteObjectVersion" + S3_ACTION_GET_OBJECT_VERSION = "s3:GetObjectVersion" + S3_ACTION_GET_OBJECT_ATTRIBUTES = "s3:GetObjectAttributes" + S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES = "s3:GetObjectVersionAttributes" // Object ACL operations S3_ACTION_GET_OBJECT_ACL = "s3:GetObjectAcl" S3_ACTION_PUT_OBJECT_ACL = "s3:PutObjectAcl" + // Version-specific ACL operations (?acl&versionId) + S3_ACTION_GET_OBJECT_VERSION_ACL = "s3:GetObjectVersionAcl" + S3_ACTION_PUT_OBJECT_VERSION_ACL = "s3:PutObjectVersionAcl" + // Object tagging operations S3_ACTION_GET_OBJECT_TAGGING = "s3:GetObjectTagging" S3_ACTION_PUT_OBJECT_TAGGING = "s3:PutObjectTagging" diff --git a/weed/s3api/s3_constants/s3_actions.go b/weed/s3api/s3_constants/s3_actions.go index d90b8cfd32..c920e87cbe 100644 --- a/weed/s3api/s3_constants/s3_actions.go +++ b/weed/s3api/s3_constants/s3_actions.go @@ -29,6 +29,7 @@ const ( SeaweedStorageDestinationHeader = "x-seaweedfs-destination" MultipartUploadsFolder = ".uploads" VersionsFolder = ".versions" + SystemFolder = ".system" FolderMimeType = "httpd/unix-directory" DirectoryMimeType = "application/x-directory" ) diff --git a/weed/s3api/s3_existing_object_tag_test.go b/weed/s3api/s3_existing_object_tag_test.go index b3cb54f101..c1809d9573 100644 --- a/weed/s3api/s3_existing_object_tag_test.go +++ b/weed/s3api/s3_existing_object_tag_test.go @@ -576,3 +576,97 @@ func TestRecheckObjectAclTagConditionedPolicy(t *testing.T) { assert.Equal(t, s3err.ErrNone, s3a.recheckPolicyWithObjectEntry(putReq, "b", "secret.txt", s3_constants.S3_ACTION_PUT_OBJECT_ACL, publicEntry, "test")) } + +// A versioned GetObjectAttributes request is authorized against the version +// read grant, not the attributes action the original request resolves to: an +// explicit bucket-policy Deny on s3:GetObjectVersion blocks it, while a legacy +// identity holding only the coarse Read grant is still accepted. +func TestCheckVersionReadGrant(t *testing.T) { + s3a := &S3ApiServer{ + policyEngine: NewBucketPolicyEngine(), + iam: &IdentityAccessManagement{}, + } + + denyPolicyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": ["arn:aws:s3:::deny-bucket", "arn:aws:s3:::deny-bucket/*"] + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:GetObjectVersion", + "Resource": "arn:aws:s3:::deny-bucket/key" + } + ] + }` + require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("deny-bucket", denyPolicyJSON)) + + allowPolicyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": ["arn:aws:s3:::allow-bucket", "arn:aws:s3:::allow-bucket/*"] + } + ] + }` + require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("allow-bucket", allowPolicyJSON)) + + identity := &Identity{ + Name: "reader", + Actions: []Action{s3_constants.ACTION_READ}, + PrincipalArn: "arn:aws:iam::000000000000:user/reader", + } + + attrsReq := func(bucket string) *http.Request { + req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/key?attributes&versionId=v1", nil) + return req.WithContext(s3_constants.SetIdentityInContext(req.Context(), identity)) + } + + assert.Equal(t, s3err.ErrAccessDenied, + s3a.checkVersionReadGrant(attrsReq("deny-bucket"), "deny-bucket", "key", "v1", nil)) + assert.Equal(t, s3err.ErrNone, + s3a.checkVersionReadGrant(attrsReq("allow-bucket"), "allow-bucket", "key", "v1", nil)) + assert.Equal(t, s3err.ErrNone, + s3a.checkVersionReadGrant(attrsReq("no-policy-bucket"), "no-policy-bucket", "key", "v1", nil)) +} + +// Presigned-URL credentials ride along on the synthesized version-read +// request, so an s3:authType=REST-QUERY-STRING policy condition decides the +// recheck the same way it decided the real request. +func TestCheckVersionReadGrantPreservesQueryAuth(t *testing.T) { + s3a := &S3ApiServer{ + policyEngine: NewBucketPolicyEngine(), + iam: &IdentityAccessManagement{}, + } + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": ["arn:aws:s3:::qs-bucket", "arn:aws:s3:::qs-bucket/*"], + "Condition": {"StringEquals": {"s3:authType": "REST-QUERY-STRING"}} + }] + }` + require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("qs-bucket", policyJSON)) + + presigned := httptest.NewRequest(http.MethodGet, + "/qs-bucket/key?attributes&versionId=v1&AWSAccessKeyId=AK&Signature=sig&Expires=9999999999", nil) + assert.Equal(t, s3err.ErrNone, + s3a.checkVersionReadGrant(presigned, "qs-bucket", "key", "v1", nil)) + + keyReq := objectKeyPolicyRequest(presigned, http.MethodGet, "qs-bucket", "key", "v1") + q := keyReq.URL.Query() + assert.Equal(t, "AK", q.Get("AWSAccessKeyId")) + assert.Equal(t, "v1", q.Get("versionId")) + assert.False(t, q.Has("attributes")) +} diff --git a/weed/s3api/s3api_bucket_owner_index.go b/weed/s3api/s3api_bucket_owner_index.go index b69cbc66a2..a8ccd8ae31 100644 --- a/weed/s3api/s3api_bucket_owner_index.go +++ b/weed/s3api/s3api_bucket_owner_index.go @@ -27,7 +27,7 @@ import ( // indexes pre-existing buckets. The .complete marker is written once the // backfill has finished; until it exists readers keep scanning /buckets. const ( - s3SystemFolder = ".system" + s3SystemFolder = s3_constants.SystemFolder bucketOwnerIndexFolder = "owners" bucketOwnerIndexReadyMarker = ".complete" ) diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index bce22df8aa..4b837117f4 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -2609,8 +2609,8 @@ func (e *EmbeddedIamApi) handleImplicitUsername(r *http.Request, values url.Valu } } -// iamSelfServiceActions are actions that users can perform on their own resources without admin rights. -// According to AWS IAM, users can manage their own access keys without requiring full admin permissions. +// iamSelfServiceActions are actions that users can perform on their own resources without admin rights, +// unless selfServiceRequiresGrant is set, in which case they need an iam: grant like any other. var iamSelfServiceActions = map[string]bool{ "CreateAccessKey": true, "DeleteAccessKey": true, @@ -2656,15 +2656,18 @@ func iamTargetUserName(action string, r *http.Request) string { // data-plane policy, which says nothing about administering the credential // store. // -// Users may run self-service actions against their own identity without any -// IAM grant, matching AWS. +// By default users may run self-service actions against their own identity +// without any IAM grant. With selfServiceRequiresGrant set they need the +// iam: grant as AWS requires, authorized against the target user's +// ARN (the caller's own when UserName is empty), so a policy can confine it +// to the caller's own keys. func (iam *IdentityAccessManagement) AuthorizeIamAction(r *http.Request, identity *Identity, action, targetUserName string) s3err.ErrorCode { // The anonymous identity has no user of its own, so every self-service // action it names would run against someone else's. if identity == nil || identity.Name == s3_constants.AccountAnonymousId { return s3err.ErrAccessDenied } - if iamRequiresAdminForOthers(action) && (targetUserName == "" || targetUserName == identity.Name) { + if !iam.selfServiceRequiresGrant && iamRequiresAdminForOthers(action) && (targetUserName == "" || targetUserName == identity.Name) { return s3err.ErrNone } if identity.isAdmin() { @@ -2677,6 +2680,13 @@ func (iam *IdentityAccessManagement) AuthorizeIamAction(r *http.Request, identit if iamRequiresSelfTarget(action) && targetUserName != "" { resource = fmt.Sprintf("arn:aws:iam:::user/%s", targetUserName) } + if iam.selfServiceRequiresGrant && iamRequiresAdminForOthers(action) { + target := targetUserName + if target == "" { + target = identity.Name + } + resource = fmt.Sprintf("arn:aws:iam:::user/%s", target) + } return iam.VerifyActionPermission(r, identity, Action("iam:"+action), resource, "") } diff --git a/weed/s3api/s3api_embedded_iam_authz_test.go b/weed/s3api/s3api_embedded_iam_authz_test.go index 7f4725fab9..81a1dd3e8b 100644 --- a/weed/s3api/s3api_embedded_iam_authz_test.go +++ b/weed/s3api/s3api_embedded_iam_authz_test.go @@ -145,3 +145,75 @@ func TestAuthorizeIamActionCreateServiceAccountScopedByPolicyResource(t *testing assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), admin, "CreateServiceAccount", "victim")) } + +var iamSelfServiceActionNames = []string{"CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey", "ListAccessKeys", "GetUser"} + +func ownKeysPolicy(resource string) string { + return `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["iam:CreateAccessKey","iam:DeleteAccessKey","iam:UpdateAccessKey","iam:ListAccessKeys","iam:GetUser"],"Resource":["` + resource + `"]}]}` +} + +// With selfServiceRequiresGrant, an identity holding only data-plane rights may +// no longer manage its own access keys: a leaked key cannot mint further keys +// that outlive its own revocation, nor deactivate the keys it was issued. +func TestAuthorizeIamActionSelfServiceRequiresGrantWhenConfigured(t *testing.T) { + iam := newIamAuthzTestIam(t) + iam.selfServiceRequiresGrant = true + powerUser := &Identity{Name: "power_user", PolicyNames: []string{"PowerUserPolicy"}} + bare := &Identity{Name: "bare_user"} + + for _, action := range iamSelfServiceActionNames { + for _, identity := range []*Identity{powerUser, bare} { + for _, target := range []string{"", identity.Name, "victim"} { + assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), identity, action, target), + "%s by %s on %q", action, identity.Name, target) + } + } + } +} + +// The grant is authorized against the target user's ARN, the caller's own when +// UserName is empty, so a policy can confine it to the caller's own keys. +func TestAuthorizeIamActionSelfServiceGrantIsScopedByResource(t *testing.T) { + for _, resource := range []string{"arn:aws:iam:::user/key_owner", "arn:aws:iam:::user/${aws:username}"} { + t.Run(resource, func(t *testing.T) { + iam := newIamAuthzTestIam(t) + iam.selfServiceRequiresGrant = true + require.NoError(t, iam.PutPolicy("OwnKeysPolicy", ownKeysPolicy(resource))) + owner := &Identity{Name: "key_owner", PolicyNames: []string{"OwnKeysPolicy"}} + + for _, action := range iamSelfServiceActionNames { + assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, ""), "%s on itself, implicit", action) + assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, "key_owner"), "%s on itself", action) + assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), owner, action, "victim"), "%s on another user", action) + } + assert.Equal(t, s3err.ErrAccessDenied, iam.AuthorizeIamAction(iamPostRequest(""), owner, "CreateUser", "victim")) + }) + } +} + +// The setting changes nothing for the admin, nor for an iam:* grant. +func TestAuthorizeIamActionSelfServiceRequiresGrantKeepsAdmin(t *testing.T) { + iam := newIamAuthzTestIam(t) + iam.selfServiceRequiresGrant = true + admin := &Identity{Name: "admin", Actions: []Action{s3_constants.ACTION_ADMIN}} + operator := &Identity{Name: "iam_operator", PolicyNames: []string{"IamAdminPolicy"}} + + for _, action := range iamSelfServiceActionNames { + for _, identity := range []*Identity{admin, operator} { + for _, target := range []string{"", identity.Name, "victim"} { + assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), identity, action, target), + "%s by %s on %q", action, identity.Name, target) + } + } + } +} + +// Off by default: self-service stays allowed without a grant. +func TestAuthorizeIamActionSelfServiceDefaultUnchanged(t *testing.T) { + iam := newIamAuthzTestIam(t) + powerUser := &Identity{Name: "power_user", PolicyNames: []string{"PowerUserPolicy"}} + for _, action := range iamSelfServiceActionNames { + assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), powerUser, action, ""), action) + assert.Equal(t, s3err.ErrNone, iam.AuthorizeIamAction(iamPostRequest(""), powerUser, action, "power_user"), action) + } +} diff --git a/weed/s3api/s3api_head_bucket_test.go b/weed/s3api/s3api_head_bucket_test.go index 0c5cbf82ad..d83308c57b 100644 --- a/weed/s3api/s3api_head_bucket_test.go +++ b/weed/s3api/s3api_head_bucket_test.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httptest" "path" - "strings" "testing" "github.com/gorilla/mux" @@ -56,19 +55,16 @@ func TestHeadBucketSeparatesLookupFailureFromMissingBucket(t *testing.T) { name string filer *fakeLookupFiler wantCode int - wantBody string }{ { name: "transient lookup failure", filer: &fakeLookupFiler{lookupErr: status.Error(codes.Internal, "filer store unavailable")}, wantCode: http.StatusInternalServerError, - wantBody: "InternalError", }, { name: "missing bucket", filer: &fakeLookupFiler{}, wantCode: http.StatusNotFound, - wantBody: "NoSuchBucket", }, { name: "existing bucket", @@ -88,8 +84,10 @@ func TestHeadBucketSeparatesLookupFailureFromMissingBucket(t *testing.T) { if rr.Code != tc.wantCode { t.Fatalf("status = %d, want %d: %s", rr.Code, tc.wantCode, rr.Body.String()) } - if tc.wantBody != "" && !strings.Contains(rr.Body.String(), tc.wantBody) { - t.Fatalf("body = %s, want %s", rr.Body.String(), tc.wantBody) + // HEAD returns the status without an XML error body, including for + // lookup failures and missing buckets. + if rr.Body.Len() != 0 { + t.Fatalf("HEAD response must have an empty body, got %q", rr.Body.String()) } }) } diff --git a/weed/s3api/s3api_iceberg_credentials.go b/weed/s3api/s3api_iceberg_credentials.go index 830a2ca98f..50af7cdb99 100644 --- a/weed/s3api/s3api_iceberg_credentials.go +++ b/weed/s3api/s3api_iceberg_credentials.go @@ -2,12 +2,15 @@ package s3api import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "strings" "time" "github.com/seaweedfs/seaweedfs/weed/iam/sts" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" ) // icebergVendedCredentialsProvider names the origin recorded on sessions the @@ -34,6 +37,11 @@ type VendedCredentials struct { // has already authenticated and authorized. Returns nil when no role is // configured, which leaves the catalog telling the client to keep using its // own credentials. +// +// The access key the caller authenticated with (from ctx) is recorded in the +// session: a short hash of it ends the session name, so the audit log's +// requester_arn tells apart the holders of several keys on one identity, and +// the key id itself is carried in the session token. func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) { if s3a.icebergCredentialRole == "" { return nil, nil @@ -55,12 +63,14 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc return nil, err } + accessKey := s3_constants.AccessKeyIdFromContext(ctx) request := &sts.AssumeRoleForPrincipalRequest{ - RoleArn: s3a.icebergCredentialRole, - Principal: principal, - RoleSessionName: icebergSessionName(principal, bucket, prefix), - ProviderName: icebergVendedCredentialsProvider, - Policy: &policy, + RoleArn: s3a.icebergCredentialRole, + Principal: principal, + RoleSessionName: icebergSessionName(principal, bucket, prefix, accessKey), + ProviderName: icebergVendedCredentialsProvider, + Policy: &policy, + SourceAccessKeyId: accessKey, } if s3a.icebergCredentialDuration > 0 { duration := s3a.icebergCredentialDuration @@ -162,9 +172,25 @@ func tablePrefixSessionPolicy(bucket, prefix string, readOnly bool) (string, err return string(encoded), nil } -// icebergSessionName builds a session name that identifies the caller and the -// table in audit logs, within the 64-character AWS limit. -func icebergSessionName(principal, bucket, prefix string) string { +// icebergSessionKeyTagLength is the hex length of the access key hash that ends +// a vended session's name: 48 bits, so two keys of one identity collide with +// odds of about 2^-48. +const icebergSessionKeyTagLength = 12 + +// icebergSessionKeyTag is "-k" and the first icebergSessionKeyTagLength hex +// digits of the SHA-256 of the access key id, or "" without a key. +func icebergSessionKeyTag(accessKey string) string { + if accessKey == "" { + return "" + } + sum := sha256.Sum256([]byte(accessKey)) + return "-k" + hex.EncodeToString(sum[:])[:icebergSessionKeyTagLength] +} + +// icebergSessionName builds a session name that identifies the caller, the +// table and the caller's access key in audit logs, within the 64-character AWS +// limit. The key tag is never cut: the table part gives way to it. +func icebergSessionName(principal, bucket, prefix, accessKey string) string { name := fmt.Sprintf("iceberg-%s-%s", principal, strings.ReplaceAll(strings.Trim(prefix, "/"), "/", "-")) if strings.Trim(prefix, "/") == "" { name = fmt.Sprintf("iceberg-%s-%s", principal, bucket) @@ -177,8 +203,9 @@ func icebergSessionName(principal, bucket, prefix string) string { return '-' } }, name) - if len(name) > 64 { - name = name[:64] + tag := icebergSessionKeyTag(accessKey) + if limit := 64 - len(tag); len(name) > limit { + name = name[:limit] } - return name + return name + tag } diff --git a/weed/s3api/s3api_iceberg_credentials_test.go b/weed/s3api/s3api_iceberg_credentials_test.go index 77df42e8b6..bdca1bf5da 100644 --- a/weed/s3api/s3api_iceberg_credentials_test.go +++ b/weed/s3api/s3api_iceberg_credentials_test.go @@ -114,7 +114,7 @@ func TestTablePrefixSessionPolicyLeavesBucketLocationUnconditioned(t *testing.T) } func TestIcebergSessionNameIsBounded(t *testing.T) { - name := icebergSessionName(strings.Repeat("principal", 10), "warehouse", "ns/table") + name := icebergSessionName(strings.Repeat("principal", 10), "warehouse", "ns/table", "") if len(name) > 64 { t.Errorf("session name is %d chars, want at most 64", len(name)) } diff --git a/weed/s3api/s3api_iceberg_session_name_test.go b/weed/s3api/s3api_iceberg_session_name_test.go new file mode 100644 index 0000000000..9a615bef8d --- /dev/null +++ b/weed/s3api/s3api_iceberg_session_name_test.go @@ -0,0 +1,98 @@ +package s3api + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/iam/integration" + "github.com/seaweedfs/seaweedfs/weed/iam/policy" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +func TestIcebergSessionNameWithoutKeyIsUnchanged(t *testing.T) { + assert.Equal(t, "iceberg-analytics-ns-t", icebergSessionName("analytics", "b", "ns/t", "")) + assert.Equal(t, "iceberg-anonymous-ns-t", icebergSessionName("anonymous", "b", "ns/t", "")) +} + +func TestIcebergSessionNameEndsWithTheKeyTag(t *testing.T) { + a := icebergSessionName("analytics", "b", "ns/t", sharedKeyA) + b := icebergSessionName("analytics", "b", "ns/t", sharedKeyB) + assert.NotEqual(t, a, b, "two keys of one identity must give distinguishable session names") + for _, name := range []string{a, b} { + assert.True(t, strings.HasPrefix(name, "iceberg-analytics-ns-t-k"), name) + assert.Len(t, name, len("iceberg-analytics-ns-t-k")+icebergSessionKeyTagLength) + } + assert.Equal(t, a, icebergSessionName("analytics", "b", "ns/t", sharedKeyA), "the tag is stable") + assert.NotContains(t, a, sharedKeyA, "the name carries a hash, not the key id") +} + +// A long identity and table give way to the tag; the tag is never cut. +func TestIcebergSessionNameKeepsTheWholeTagWithinTheLimit(t *testing.T) { + principal := "svc-0b6f6d1c-6f1e-4c1a-9f57-3a8f4c0f1d2e" + name := icebergSessionName(principal, "b", "a_long_namespace/a_long_table_name", sharedKeyA) + tag := icebergSessionKeyTag(sharedKeyA) + assert.LessOrEqual(t, len(name), 64) + assert.True(t, strings.HasSuffix(name, tag), name) + assert.True(t, strings.HasPrefix(name, "iceberg-"+principal), name) + for _, r := range name { + ok := r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' + assert.True(t, ok, "session name %q contains %q", name, r) + } + long := icebergSessionName(strings.Repeat("principal", 10), "b", "ns/t", sharedKeyB) + assert.Len(t, long, 64) + assert.True(t, strings.HasSuffix(long, icebergSessionKeyTag(sharedKeyB))) +} + +// The vendor reads the caller's key from the request context. +func TestVendTableCredentialsReadsTheKeyFromTheContext(t *testing.T) { + ctx := s3_constants.SetAccessKeyIdInContext(context.Background(), sharedKeyA) + assert.Equal(t, sharedKeyA, s3_constants.AccessKeyIdFromContext(ctx)) + assert.Equal(t, "", s3_constants.AccessKeyIdFromContext(context.Background())) +} + +// VendTableCredentials mints a real session from the caller's key: the session +// name ends with the key's tag and the token carries the key id. +func TestVendTableCredentialsRecordsTheCallersKey(t *testing.T) { + mgr := newTestIAMManager(t) + require.NoError(t, mgr.CreateRole(context.Background(), "", "IcebergTableAccess", &integration.RoleDefinition{ + RoleName: "IcebergTableAccess", + TrustPolicy: &policy.PolicyDocument{ + Version: "2012-10-17", + Statement: []policy.Statement{{ + Effect: "Allow", + Principal: map[string]interface{}{"Federated": icebergVendedCredentialsProvider}, + Action: []string{"sts:AssumeRoleWithCredentials"}, + }}, + }, + })) + iam := newSharedIdentityIAM() + iam.SetIAMIntegration(NewS3IAMIntegration(mgr, "")) + s3a := &S3ApiServer{iam: iam} + s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600) + + for _, key := range []string{sharedKeyA, sharedKeyB, ""} { + ctx := context.Background() + if key != "" { + // As the catalog's auth does: the holder is installed, the verifier records into it. + r := s3_constants.EnsureIdentityHolder(httptest.NewRequest(http.MethodGet, "/", nil)) + s3_constants.RecordAccessKeyId(r.Context(), key) + ctx = r.Context() + } + creds, err := s3a.VendTableCredentials(ctx, "shared-user", "warehouse", "ns/t", false) + require.NoError(t, err) + require.NotNil(t, creds) + info, err := s3a.stsService().ValidateSessionToken(context.Background(), creds.SessionToken) + require.NoError(t, err) + assert.Equal(t, key, info.SourceAccessKeyId, "key %q", key) + assert.Equal(t, icebergSessionName("shared-user", "warehouse", "ns/t", key), info.SessionName, "key %q", key) + if key != "" { + assert.Contains(t, info.SessionName, icebergSessionKeyTag(key)) + } + } +} diff --git a/weed/s3api/s3api_object_anonymous_cache_test.go b/weed/s3api/s3api_object_anonymous_cache_test.go index d5a0f2f593..a349585727 100644 --- a/weed/s3api/s3api_object_anonymous_cache_test.go +++ b/weed/s3api/s3api_object_anonymous_cache_test.go @@ -238,10 +238,12 @@ func TestAnonymousObjectACLVersionedDirectoryIdentity(t *testing.T) { f.entries["/buckets/b/folder/.versions"] = &filer_pb.Entry{IsDirectory: true, Extended: map[string][]byte{s3_constants.ExtLatestVersionIdKey: []byte("")}} } // Equivalent filer-path spellings must not recover a different object's data. - assertMissing(serveAnonymousRead(s3a, method, "folder/", route.query, headers)) + for _, object := range []string{"folder/", "folder//", "folder\\"} { + assertMissing(serveAnonymousRead(s3a, method, object, route.query, headers)) + } // Spellings the policy's folder/ grant does not cover are // masked: a miss must be indistinguishable from a denied read. - for _, object := range []string{"folder//", "folder\\"} { + for _, object := range []string{"folder//x", "folder\\x"} { rr := serveAnonymousRead(s3a, method, object, route.query, headers) require.Equal(t, http.StatusForbidden, rr.Code, rr.Body.String()) } @@ -281,3 +283,40 @@ func TestAnonymousObjectACLVersionedDirectoryIdentity(t *testing.T) { } } } + +// A literal-backslash key is a different object than its slash spelling only +// when allowBackslashInObjectKey is set. A public grant on b/folder/ must not +// authorize an anonymous read of the separate folder\ object under that mode. +func TestAnonymousObjectACLBackslashKeyIdentity(t *testing.T) { + for _, allowBackslash := range []bool{false, true} { + t.Run(fmt.Sprintf("allowBackslashInKey=%v", allowBackslash), func(t *testing.T) { + s3a, f := newAnonymousReadTestServer(t) + s3_constants.SetAllowBackslashInObjectKey(allowBackslash) + t.Cleanup(func() { s3_constants.SetAllowBackslashInObjectKey(false) }) + require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("b", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject*","Resource":"arn:aws:s3:::b/folder/"}]}`)) + slash := anonymousReadEntry([]byte(`[]`)) + slash.Name = "folder" + slash.MarkPrefixObject() + f.entries["/buckets/b/folder"] = slash + backslash := anonymousReadEntry([]byte(`[]`)) + backslash.Name = `folder\` + f.entries[`/buckets/b/folder\`] = backslash + + rr := serveAnonymousRead(s3a, http.MethodGet, "folder/", "", nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + require.Equal(t, "hello world", rr.Body.String()) + + rr = serveAnonymousRead(s3a, http.MethodGet, `folder\`, "", nil) + if !allowBackslash { + // Default mode folds the spelling onto the slash key the policy covers. + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + } else { + require.Equal(t, http.StatusForbidden, rr.Code, rr.Body.String()) + backslash.Extended[s3_constants.ExtAmzAclKey] = anonymousReadGrant("READ") + rr = serveAnonymousRead(s3a, http.MethodGet, `folder\`, "", nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + require.Equal(t, "hello world", rr.Body.String()) + } + }) + } +} diff --git a/weed/s3api/s3api_object_anonymous_read.go b/weed/s3api/s3api_object_anonymous_read.go index e5d1c21c28..6fd86001aa 100644 --- a/weed/s3api/s3api_object_anonymous_read.go +++ b/weed/s3api/s3api_object_anonymous_read.go @@ -118,7 +118,7 @@ func (s3a *S3ApiServer) authorizeAnonymousObjectRead(r *http.Request, bucket, ob // public-read bucket. Anything else masks the miss as AccessDenied. func (s3a *S3ApiServer) authorizeAnonymousBucketList(r *http.Request, bucket string) s3err.ErrorCode { identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity) - object := mux.Vars(r)["object"] + object := s3_constants.NormalizeObjectKey(mux.Vars(r)["object"]) // The caller's query parameters would otherwise retarget these checks // (?policy resolves to GetBucketPolicy) or satisfy a prefix condition on an // unrelated subtree, so each runs on a request with a fixed shape: a diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 8e747550b7..ab01cfda36 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -2251,8 +2251,9 @@ func (s3a *S3ApiServer) readChunkRange(ctx context.Context, fileId string, offse } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + err = util_http.StatusError(chunkUrl, resp) resp.Body.Close() - return nil, fmt.Errorf("unexpected status code %d for chunk %s", resp.StatusCode, fileId) + return nil, err } return resp.Body, nil diff --git a/weed/s3api/s3api_object_handlers_acl.go b/weed/s3api/s3api_object_handlers_acl.go index dc5f79eae6..e832db33dc 100644 --- a/weed/s3api/s3api_object_handlers_acl.go +++ b/weed/s3api/s3api_object_handlers_acl.go @@ -30,6 +30,10 @@ func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Reque // Check for specific version ID in query parameters versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } // Check if versioning is configured for the bucket (Enabled or Suspended) versioningConfigured, err := s3a.isVersioningConfigured(bucket) @@ -89,7 +93,11 @@ func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Reque return } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_GET_OBJECT_ACL, entry.Extended, "GetObjectAclHandler"); errCode != s3err.ErrNone { + action := s3_constants.S3_ACTION_GET_OBJECT_ACL + if versionId != "" { + action = s3_constants.S3_ACTION_GET_OBJECT_VERSION_ACL + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, action, entry.Extended, "GetObjectAclHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } @@ -142,6 +150,10 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque // Check for specific version ID in query parameters versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } // Check if versioning is configured for the bucket (Enabled or Suspended) versioningConfigured, err := s3a.isVersioningConfigured(bucket) @@ -201,7 +213,11 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque return } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_PUT_OBJECT_ACL, entry.Extended, "PutObjectAclHandler"); errCode != s3err.ErrNone { + putAction := s3_constants.S3_ACTION_PUT_OBJECT_ACL + if versionId != "" { + putAction = s3_constants.S3_ACTION_PUT_OBJECT_VERSION_ACL + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, putAction, entry.Extended, "PutObjectAclHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } diff --git a/weed/s3api/s3api_object_handlers_attributes.go b/weed/s3api/s3api_object_handlers_attributes.go index f483c13a2f..9e8881ebc1 100644 --- a/weed/s3api/s3api_object_handlers_attributes.go +++ b/weed/s3api/s3api_object_handlers_attributes.go @@ -129,6 +129,10 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt // Check for specific version ID versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } var ( entry *filer_pb.Entry @@ -219,11 +223,24 @@ func (s3a *S3ApiServer) GetObjectAttributesHandler(w http.ResponseWriter, r *htt } } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES, entry.Extended, "GetObjectAttributesHandler"); errCode != s3err.ErrNone { + action := s3_constants.S3_ACTION_GET_OBJECT_ATTRIBUTES + if versionId != "" { + action = s3_constants.S3_ACTION_GET_OBJECT_VERSION_ATTRIBUTES + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, action, entry.Extended, "GetObjectAttributesHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } + // AWS also asks for the version read grant on a versioned attributes + // request: s3:GetObjectVersionAttributes alone is not sufficient. + if versionId != "" { + if errCode := s3a.checkVersionReadGrant(r, bucket, object, versionId, entry.Extended); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + } + // Evaluate conditional headers against the resolved entry (after version resolution) // This ensures conditions are checked against the correct version, not always the latest if s3a.hasConditionalHeaders(r) { diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index fbbf2b2fd5..2473ed77cc 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -900,13 +900,13 @@ func pathToBucketObjectAndVersion(rawPath, decodedPath string) (bucket, object, } bucket, object = pathToBucketAndObject(pathForBucket) - return bucket, object, versionId + return bucket, s3_constants.NormalizeObjectKey(object), versionId } } } bucket, object = pathToBucketAndObject(pathForBucket) - return bucket, object, versionId + return bucket, s3_constants.NormalizeObjectKey(object), versionId } type CopyPartResult struct { diff --git a/weed/s3api/s3api_object_handlers_delete.go b/weed/s3api/s3api_object_handlers_delete.go index 3817a28adc..caa0069624 100644 --- a/weed/s3api/s3api_object_handlers_delete.go +++ b/weed/s3api/s3api_object_handlers_delete.go @@ -479,29 +479,44 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h deleteErrors = append(deleteErrors, deleteErrorFromCode(validationCode, object.Key, object.VersionId)) continue } - if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, object.Key); err != nil { + key := s3_constants.NormalizeObjectKey(object.Key) + if key == "" { + // A key of only slashes can never name a stored entry; like a + // missing key the delete is an idempotent no-op, and a + // conditional delete has no target to match. + if object.ETag != "" { + deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrNoSuchKey, object.Key, object.VersionId)) + continue + } + deletedCount++ + if !deleteObjects.Quiet { + deletedObjects = append(deletedObjects, ObjectIdentifier{Key: object.Key, VersionId: object.VersionId}) + } + continue + } + if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, key); err != nil { deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrAccessDenied, object.Key, object.VersionId)) continue } - if authErr := s3a.iam.AuthorizeObjectDelete(r, identity, bucket, object.Key, object.VersionId); authErr != s3err.ErrNone { + if authErr := s3a.iam.AuthorizeObjectDelete(r, identity, bucket, key, object.VersionId); authErr != s3err.ErrNone { // Defer to the recheck, mirroring the copy-source fallback: a // tag-conditioned Allow may still grant what phase one denied. - if authErr != s3err.ErrAccessDenied || s3a.recheckPolicyForKeyWithLookup(r, http.MethodDelete, bucket, object.Key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler") != s3err.ErrNone { + if authErr != s3err.ErrAccessDenied || s3a.recheckPolicyForKeyWithLookup(r, http.MethodDelete, bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler") != s3err.ErrNone { deleteErrors = append(deleteErrors, deleteErrorFromCode(authErr, object.Key, object.VersionId)) continue } } - if errCode := s3a.recheckTagConditionedPolicyWithLookup(objectKeyPolicyRequest(r, http.MethodDelete, bucket, object.Key, object.VersionId), bucket, object.Key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler"); errCode != s3err.ErrNone { + if errCode := s3a.recheckTagConditionedPolicyWithLookup(objectKeyPolicyRequest(r, http.MethodDelete, bucket, key, object.VersionId), bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler"); errCode != s3err.ErrNone { deleteErrors = append(deleteErrors, deleteErrorFromCode(errCode, object.Key, object.VersionId)) continue } var deleteResult deleteMutationResult - deleteCode := s3a.withObjectWriteLock(bucket, object.Key, func() s3err.ErrorCode { - return s3a.checkDeleteIfMatch(bucket, object.Key, object.VersionId, versioningState, object.ETag, s3err.ErrNoSuchKey) + deleteCode := s3a.withObjectWriteLock(bucket, key, func() s3err.ErrorCode { + return s3a.checkDeleteIfMatch(bucket, key, object.VersionId, versioningState, object.ETag, s3err.ErrNoSuchKey) }, func() s3err.ErrorCode { if versioningConfigured { - result, errCode := s3a.deleteVersionedObject(r, bucket, object.Key, object.VersionId, versioningState) + result, errCode := s3a.deleteVersionedObject(r, bucket, key, object.VersionId, versioningState) if errCode != s3err.ErrNone { return errCode } @@ -509,18 +524,18 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h return s3err.ErrNone } - governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key) - if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, "", governanceBypassAllowed); err != nil { - glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s: %v", bucket, object.Key, err) + governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, key) + if err := s3a.enforceObjectLockProtections(r, bucket, key, "", governanceBypassAllowed); err != nil { + glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s: %v", bucket, key, err) return s3err.ErrAccessDenied } - if strings.HasSuffix(object.Key, "/") { - return s3a.deleteDirectoryMarker(r, bucket, object.Key) + if strings.HasSuffix(key, "/") { + return s3a.deleteDirectoryMarker(r, bucket, key) } - if err := s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, object.Key, false); err != nil { - glog.Errorf("DeleteMultipleObjectsHandler: failed to delete %s/%s: %v", bucket, object.Key, err) + if err := s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, key, false); err != nil { + glog.Errorf("DeleteMultipleObjectsHandler: failed to delete %s/%s: %v", bucket, key, err) return s3err.ErrInternalError } diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 679c9f438a..6b049e0e70 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -538,6 +538,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq if excludedKey != "" && !entry.IsDirectory && fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):] == excludedKey { return } + // A delimiter marker excludes the entire folded prefix, not just + // the key bearing its name. This also applies to delimiters such as + // "_", whose groups are not directory blocks in the filer walk. + if delimiter != "" { + name := entry.Name + if entry.IsDirectory && !entry.IsPrefixObject() { + name += "/" + } + if folded := prefixForKey(dir, name); folded != "" && folded <= requestMarker { + return + } + } dirName, entryName, _ := entryUrlEncode(dir, entry.Name, encodingTypeUrl) if entry.IsDirectory { if originalPrefix != "" { @@ -1170,13 +1182,12 @@ func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb. // we must stop processing siblings to avoid matching unrelated entries like "foo1000". matchedPrefixDir := cursor.prefixEndsOnDelimiter && entry.Name == prefix && entry.IsDirectory if cursor.prefixEndsOnDelimiter { - if entry.Name == prefix && entry.IsDirectory { - if delimiter != "/" { - cursor.prefixEndsOnDelimiter = false - } - } else { + if !matchedPrefixDir { continue } + // Keep the exact-directory filter until emitDir descends into + // the deferred block. Clearing it at the name position lets + // siblings like "a.txt" leak into a listing for "a/". } if entry.IsDirectory { // glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys) diff --git a/weed/s3api/s3api_object_handlers_list_regression_test.go b/weed/s3api/s3api_object_handlers_list_regression_test.go new file mode 100644 index 0000000000..72d4066189 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_list_regression_test.go @@ -0,0 +1,125 @@ +package s3api + +import ( + "context" + "fmt" + "net/url" + "sort" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Exercise the response builder as well as the directory walker: deferred +// directory resume markers can make the outer loop repeat prefixes. +// Derive the expected response from S3 keys, independently of filer traversal. +func TestListFilerEntriesListingRegression(t *testing.T) { + keys := []string{ + "a", "a-c", "a.txt", "a/b", "a/c/d", "a0", + "boo/bar", "boo/baz", "boo/boo", "boo/boo2", + "foo_bar", "foo_bar_baz", "foo_baz", "logs/", "quux ab", "space dir/value", "z", + } + sort.Strings(keys) + prefixObject := newDir("a") + prefixObject.MarkPrefixObject() + file := func(name string) *filer_pb.Entry { + return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}} + } + filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{ + "/buckets/test": { + prefixObject, file("a-c"), file("a.txt"), file("a0"), newDir("boo"), + file("foo_bar"), file("foo_bar_baz"), file("foo_baz"), + directoryMarker("logs"), file("quux ab"), newDir("space dir"), file("z"), + }, + "/buckets/test/a": {file("b"), newDir("c")}, + "/buckets/test/a/c": {file("d")}, + "/buckets/test/boo": {file("bar"), file("baz"), file("boo"), file("boo2")}, + "/buckets/test/logs": {}, + "/buckets/test/space dir": {file("value")}, + }} + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.option.BucketsPath = "/buckets" + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + for _, versioning := range []string{"", "Enabled", "Suspended"} { + t.Run("versioning="+versioning, func(t *testing.T) { + s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test", Versioning: versioning}) + for _, encoded := range []bool{false, true} { + t.Run(fmt.Sprintf("encoded=%t", encoded), func(t *testing.T) { + testListingRegressionMatrix(t, s3a, keys, encoded) + }) + } + }) + } +} + +func testListingRegressionMatrix(t *testing.T, s3a *S3ApiServer, keys []string, encoded bool) { + t.Helper() + for _, prefix := range []string{"", "a", "a/", "a/c/", "boo", "boo/", "boo/b", "foo", "foo_", "logs/", "space dir/", "missing/"} { + for _, delimiter := range []string{"", "/", "_"} { + for _, pageSize := range []uint16{1, 2, 1000} { + t.Run(fmt.Sprintf("prefix=%s/delimiter=%s/max=%d", prefix, delimiter, pageSize), func(t *testing.T) { + var wantKeys, wantPrefixes []string + prefixes := map[string]bool{} + for _, key := range keys { + if !strings.HasPrefix(key, prefix) { + continue + } + if delimiter != "" { + if i := strings.Index(strings.TrimPrefix(key, prefix), delimiter); i >= 0 { + prefixes[key[:len(prefix)+i+len(delimiter)]] = true + continue + } + } + wantKeys = append(wantKeys, key) + } + for p := range prefixes { + wantPrefixes = append(wantPrefixes, p) + } + sort.Strings(wantKeys) + sort.Strings(wantPrefixes) + + var gotKeys, gotPrefixes []string + marker := "" + for page := 0; ; page++ { + require.Less(t, page, len(keys)+1, "listing did not terminate") + response, err := s3a.listFilerEntries(context.Background(), listObjectsRequest{ + bucket: "test", prefix: prefix, delimiter: delimiter, + marker: adjustMarkerForDelimiter(marker, prefix, delimiter), + requestMarker: marker, maxKeys: pageSize, + encodingTypeUrl: encoded, + }) + require.NoError(t, err) + decode := func(value string) string { + if !encoded { + return value + } + decoded, err := url.QueryUnescape(value) + require.NoError(t, err) + return decoded + } + for _, entry := range response.Contents { + gotKeys = append(gotKeys, decode(entry.Key)) + } + for _, entry := range response.CommonPrefixes { + gotPrefixes = append(gotPrefixes, decode(entry.Prefix)) + } + if !response.IsTruncated { + break + } + require.Greater(t, response.NextMarker, marker, "pagination must advance") + marker = response.NextMarker + } + // Sorting retains duplicates, so repeated prefixes still fail. + sort.Strings(gotKeys) + sort.Strings(gotPrefixes) + assert.Equal(t, wantKeys, gotKeys) + assert.Equal(t, wantPrefixes, gotPrefixes) + }) + } + } + } +} diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index 3d5cffbc8e..4fb604abbf 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -2,6 +2,7 @@ package s3api import ( "bytes" + "context" "encoding/base64" "errors" "fmt" @@ -80,7 +81,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R } // Verify policy signature. - identity, errCode := s3a.iam.doesPolicySignatureMatch(formValues) + identity, errCode := s3a.iam.doesPolicySignatureMatch(r.Context(), formValues) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode)) return @@ -362,18 +363,19 @@ func getRedirectPostRawQuery(bucket, key, etag string) string { } // Check to see if Policy is signed correctly, returning the signing identity. -// The identity's Write permission on the bucket is checked only once the -// signature verifies, as on the header-signed path, so a form that fails -// verification learns nothing about the key's permissions. A verified identity -// that may not write is returned with ErrAccessDenied. -func (iam *IdentityAccessManagement) doesPolicySignatureMatch(formValues http.Header) (*Identity, s3err.ErrorCode) { +// The verified form's access key is recorded in ctx's identity holder. The +// identity's Write permission on the bucket is checked only once the signature +// verifies, as on the header-signed path, so a form that fails verification +// learns nothing about the key's permissions. A verified identity that may not +// write is returned with ErrAccessDenied. +func (iam *IdentityAccessManagement) doesPolicySignatureMatch(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) { var identity *Identity var errCode s3err.ErrorCode // For SignV2 - Signature field will be valid if _, ok := formValues["Signature"]; ok { - identity, errCode = iam.doesPolicySignatureV2Match(formValues) + identity, errCode = iam.doesPolicySignatureV2Match(ctx, formValues) } else { - identity, errCode = iam.doesPolicySignatureV4Match(formValues) + identity, errCode = iam.doesPolicySignatureV4Match(ctx, formValues) } if errCode != s3err.ErrNone { return nil, errCode diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 1b18f0c560..b38a205f16 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -696,6 +696,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader MimeType: r.Header.Get("Content-Type"), Cipher: s3a.cipher, // encrypt data on volume servers AssignFunc: assignFunc, + LookupJwtFunc: s3a.lookupVolumeJwt, }) if err != nil { glog.Errorf("putToFiler: chunked upload failed: %v", err) @@ -3022,6 +3023,15 @@ func (s3a *S3ApiServer) checkConditionalHeadersForReads(r *http.Request, bucket, return s3a.validateConditionalHeadersForReads(r, headers, entry, bucket, object) } +// lookupVolumeJwt mints a fresh volume-server JWT for an existing file id, so a +// cleanup delete that runs past the assignment JWT's expiry still authenticates. +func (s3a *S3ApiServer) lookupVolumeJwt(ctx context.Context, fileId string) security.EncodedJwt { + if s3a.masterClient == nil { + return "" + } + return operation.LookupJwt(ctx, s3a.masterClient.GetMaster(ctx), s3a.option.GrpcDialOption, fileId) +} + // deleteOrphanedChunks attempts to delete chunks that were uploaded but whose entry creation failed // This prevents resource leaks and wasted storage. Errors are logged but don't prevent cleanup attempts. func (s3a *S3ApiServer) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) { diff --git a/weed/s3api/s3api_object_handlers_tagging.go b/weed/s3api/s3api_object_handlers_tagging.go index 37761322f8..466d8bcfd4 100644 --- a/weed/s3api/s3api_object_handlers_tagging.go +++ b/weed/s3api/s3api_object_handlers_tagging.go @@ -29,6 +29,10 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R // Check for specific version ID in query parameters versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } // Check if versioning is configured for the bucket (Enabled or Suspended) versioningConfigured, err := s3a.isVersioningConfigured(bucket) @@ -87,7 +91,11 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R } } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_GET_OBJECT_TAGGING, entry.Extended, "GetObjectTaggingHandler"); errCode != s3err.ErrNone { + action := s3_constants.S3_ACTION_GET_OBJECT_TAGGING + if versionId != "" { + action = s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, action, entry.Extended, "GetObjectTaggingHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } @@ -140,6 +148,10 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R // Check for specific version ID in query parameters versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } // Check if versioning is configured for the bucket (Enabled or Suspended) versioningConfigured, err := s3a.isVersioningConfigured(bucket) @@ -185,7 +197,11 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R target := util.FullPath(fmt.Sprintf("%s/%s", s3a.bucketDir(bucket), object)) dir, name := target.DirAndName() - if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, s3_constants.S3_ACTION_PUT_OBJECT_TAGGING, "PutObjectTaggingHandler"); errCode != s3err.ErrNone { + putAction := s3_constants.S3_ACTION_PUT_OBJECT_TAGGING + if versionId != "" { + putAction = s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING + } + if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, putAction, "PutObjectTaggingHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } @@ -206,7 +222,11 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R return } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_PUT_OBJECT_TAGGING, entry.Extended, "PutObjectTaggingHandler"); errCode != s3err.ErrNone { + putAction := s3_constants.S3_ACTION_PUT_OBJECT_TAGGING + if versionId != "" { + putAction = s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, putAction, entry.Extended, "PutObjectTaggingHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } @@ -268,6 +288,10 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt // Check for specific version ID in query parameters versionId := r.URL.Query().Get("versionId") + if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } // Check if versioning is configured for the bucket (Enabled or Suspended) versioningConfigured, err := s3a.isVersioningConfigured(bucket) @@ -313,7 +337,11 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt target := util.FullPath(fmt.Sprintf("%s/%s", s3a.bucketDir(bucket), object)) dir, name := target.DirAndName() - if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING, "DeleteObjectTaggingHandler"); errCode != s3err.ErrNone { + deleteAction := s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING + if versionId != "" { + deleteAction = s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING + } + if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, deleteAction, "DeleteObjectTaggingHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } @@ -335,7 +363,11 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt return } - if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING, entry.Extended, "DeleteObjectTaggingHandler"); errCode != s3err.ErrNone { + deleteAction := s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING + if versionId != "" { + deleteAction = s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING + } + if errCode := s3a.recheckPolicyWithObjectEntry(r, bucket, object, deleteAction, entry.Extended, "DeleteObjectTaggingHandler"); errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index f7c0e477cf..86b1025081 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -78,6 +78,14 @@ type S3ApiServerOption struct { // AllowUntrustedRemoteEndpoints lets a read of a remote-only object dial a // mounted endpoint that resolves to a loopback / private / metadata host. AllowUntrustedRemoteEndpoints bool + // AllowBackslashInKey keeps '\' a literal object-key character instead of + // normalizing it to '/', matching AWS S3 where the two are distinct keys. + AllowBackslashInKey bool + // IamSelfServiceRequiresGrant makes managing one's own access keys + // (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, + // GetUser) need an explicit iam: grant instead of being allowed to every + // authenticated identity. + IamSelfServiceRequiresGrant bool } // s3ChunkCacheChunkSizeMB is the assumed chunk size (in MiB) used to convert @@ -97,6 +105,7 @@ type S3ApiServer struct { randomClientId int32 filerGuard *security.Guard filerClient *wdclient.FilerClient + masterClient *wdclient.MasterClient client util_http_client.HTTPClientInterface bucketRegistry *BucketRegistry credentialManager *credential.CredentialManager @@ -337,6 +346,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl randomClientId: util.RandomInt32(), filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec), filerClient: filerClient, + masterClient: masterClient, cb: NewCircuitBreaker(option), credentialManager: iam.credentialManager, bucketConfigCache: NewBucketConfigCache(60 * time.Minute), // Increased TTL since cache is now event-driven @@ -532,6 +542,10 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl // pointer-to-missing-file states without waiting for a client GET. s3ApiServer.versionsReconcilerStop = s3ApiServer.startVersioningReconciler() + // Publish the key mode only once the server is fully constructed so a + // failed construction cannot flip the mode of a running server. + SetAllowBackslashInObjectKey(option.AllowBackslashInKey) + return s3ApiServer, nil } @@ -681,6 +695,21 @@ func (s3a *S3ApiServer) recheckPolicyWithObjectEntry(r *http.Request, bucket, ob return errCode } +// checkVersionReadGrant enforces the version read grant AWS requires alongside +// a version-scoped action such as s3:GetObjectVersionAttributes. The check runs +// on a request shaped like a plain versioned GET so the s3:GetObjectVersion +// action resolves from the request instead of the caller's subresource, and a +// legacy Read grant still applies. +func (s3a *S3ApiServer) checkVersionReadGrant(r *http.Request, bucket, object, versionId string, objectEntry map[string][]byte) s3err.ErrorCode { + versionReadReq := objectKeyPolicyRequest(r, http.MethodGet, bucket, object, versionId) + if identity, ok := GetIdentityFromContext(r).(*Identity); ok && identity != nil { + if errCode := s3a.iam.VerifyActionPermission(versionReadReq, identity, ACTION_READ, bucket, object); errCode != s3err.ErrNone { + return errCode + } + } + return s3a.recheckPolicyWithObjectEntry(versionReadReq, bucket, object, ACTION_READ, objectEntry, "checkVersionReadGrant") +} + // recheckCopySourcePolicy repeats the source object's policy check once its // entry is fetched, so conditions on the object's existing tags apply to a // CopyObject / UploadPartCopy / RenameObject source the same way they do to a @@ -894,6 +923,9 @@ func (s3a *S3ApiServer) UnifiedPostHandler(w http.ResponseWriter, r *http.Reques func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // runs before subrouter matching so Queries() matchers see the fixed query router.Use(util_http.EscapeSemicolonsInQuery) + // a request naming two subresources routes one way and authorizes another; + // reject it rather than guess + router.Use(rejectAmbiguousSubresource) // API Router apiRouter := router.PathPrefix("/").Subrouter() apiRouter.Use(request_id.Middleware) diff --git a/weed/s3api/s3api_sts_web_identity_http_test.go b/weed/s3api/s3api_sts_web_identity_http_test.go index 1c045261c3..ff05a11d9c 100644 --- a/weed/s3api/s3api_sts_web_identity_http_test.go +++ b/weed/s3api/s3api_sts_web_identity_http_test.go @@ -7,10 +7,12 @@ import ( "encoding/base64" "encoding/json" "encoding/xml" + "fmt" "net/http" "net/http/httptest" "net/url" "strings" + "sync/atomic" "testing" "time" @@ -97,6 +99,7 @@ func TestAssumeRoleWithWebIdentityOverHTTP(t *testing.T) { iam := &IdentityAccessManagement{iamIntegration: NewS3IAMIntegration(manager, "")} handlers := NewSTSHandlers(manager.GetSTSService(), iam) + var jtiCounter atomic.Int64 signOIDCToken := func(t *testing.T, subject string) string { t.Helper() token := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ @@ -105,6 +108,10 @@ func TestAssumeRoleWithWebIdentityOverHTTP(t *testing.T) { "aud": "test-client", "exp": time.Now().Add(time.Hour).Unix(), "iat": time.Now().Unix(), + // a unique jti per token: the exchange consumes it, so two + // subtests minting the same claims in one second would otherwise + // collide on the token fingerprint and read as a replay + "jti": fmt.Sprintf("%s-%d", subject, jtiCounter.Add(1)), }) token.Header["kid"] = "web-identity-test-key" signed, err := token.SignedString(key) diff --git a/weed/s3api/s3api_subresource_ambiguity.go b/weed/s3api/s3api_subresource_ambiguity.go new file mode 100644 index 0000000000..34e326f4e3 --- /dev/null +++ b/weed/s3api/s3api_subresource_ambiguity.go @@ -0,0 +1,20 @@ +package s3api + +import ( + "net/http" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// rejectAmbiguousSubresource applies the same one-operation-per-request rule +// validateRequestPath enforces inside bucket routes to the routes outside them, +// so the check runs once for every request the router serves. +func rejectAmbiguousSubresource(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hasAmbiguousSubresource(r.URL.Query()) { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/weed/s3api/s3api_subresource_ambiguity_test.go b/weed/s3api/s3api_subresource_ambiguity_test.go new file mode 100644 index 0000000000..d461f9076a --- /dev/null +++ b/weed/s3api/s3api_subresource_ambiguity_test.go @@ -0,0 +1,56 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestRejectAmbiguousSubresource(t *testing.T) { + passthrough := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) }) + for _, query := range []string{ + "policy", + "policy=", + "tagging&versionId=v1", + "list-type=2&prefix=a&continuation-token=x", + "uploadId=abc&partNumber=3", + "uploads", + "acl&X-Amz-Signature=x&X-Amz-Expires=900", + "analytics&id=weekly", + "delete", + "versions&versionId=", + "versioning", + "prefix=a&expected-bucket-owner=123", + "tagging&tagging=x", + "", + } { + r := httptest.NewRequest(http.MethodPut, "http://localhost/bucket?"+query, nil) + w := httptest.NewRecorder() + rejectAmbiguousSubresource(passthrough).ServeHTTP(w, r) + if w.Code != http.StatusTeapot { + t.Errorf("%q should pass through, got %d", query, w.Code) + } + } + + for _, query := range []string{ + "tagging&policy", + "policy&tagging", + "list-type=2&policy", + "versions&policy", + "versioning&policy", + "delete&policy", + "cors&policy", + "acl&tagging", + "tagging&seaweedfs-quota", + "uploadId=x&acl", + "retention&legal-hold", + "ownershipControls&encryption", + } { + r := httptest.NewRequest(http.MethodPut, "http://localhost/bucket?"+query, nil) + w := httptest.NewRecorder() + rejectAmbiguousSubresource(passthrough).ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Errorf("%q should be rejected, got %d", query, w.Code) + } + } +} diff --git a/weed/s3api/s3err/audit_fluent.go b/weed/s3api/s3err/audit_fluent.go index 3985e85fa3..a8d28035be 100644 --- a/weed/s3api/s3err/audit_fluent.go +++ b/weed/s3api/s3err/audit_fluent.go @@ -29,6 +29,7 @@ type AccessLog struct { Requester string `msg:"requester" json:"requester,omitempty"` // IAM user id RequesterArn string `msg:"requester_arn" json:"requester_arn,omitempty"` // arn:aws:sts::0:assumed-role/Role/session RequesterIdentity string `msg:"requester_identity" json:"requester_identity,omitempty"` // authoritative OIDC identity claim (preferred_username/email/sub) + AccessKeyId string `msg:"access_key_id" json:"access_key_id,omitempty"` // the key the request was signed with; an STS session's temporary key id RequestID string `msg:"request_id" json:"request_id,omitempty"` // 3E57427F33A59F07 Operation string `msg:"operation" json:"operation,omitempty"` // REST.HTTP_method.resource_type REST.PUT.OBJECT Key string `msg:"key" json:"key,omitempty"` // /photos/2019/08/puppy.jpg @@ -174,6 +175,7 @@ func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *Acc Requester: s3_constants.GetIdentityNameFromContext(r), // Get from context, not header (secure) RequesterArn: s3_constants.GetPrincipalArnFromContext(r), RequesterIdentity: s3_constants.GetIdentityClaimFromContext(r), + AccessKeyId: s3_constants.GetAccessKeyIdFromContext(r), SignatureVersion: r.Header.Get(s3_constants.AmzAuthType), UserAgent: r.Header.Get("user-agent"), HostId: hostname, diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go index 72e1bd1661..d8f6b745c6 100644 --- a/weed/s3api/s3err/error_handler.go +++ b/weed/s3api/s3err/error_handler.go @@ -137,16 +137,30 @@ func setCommonHeaders(w http.ResponseWriter, r *http.Request) { } } +// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == http.StatusNoContent: + return false + case status == http.StatusNotModified: + return false + } + return true +} + func WriteResponse(w http.ResponseWriter, r *http.Request, statusCode int, response []byte, mType mimeType) { setCommonHeaders(w, r) - if response != nil { + canHaveBody := bodyAllowedForStatus(statusCode) + if response != nil && canHaveBody { w.Header().Set("Content-Length", strconv.Itoa(len(response))) } if mType != mimeNone { w.Header().Set("Content-Type", string(mType)) } w.WriteHeader(statusCode) - if response != nil { + if response != nil && canHaveBody && r.Method != http.MethodHead { glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response)) _, err := w.Write(response) if err != nil { diff --git a/weed/s3api/s3err/error_handler_test.go b/weed/s3api/s3err/error_handler_test.go index f20b8719f2..2e611dfe89 100644 --- a/weed/s3api/s3err/error_handler_test.go +++ b/weed/s3api/s3err/error_handler_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "regexp" + "strconv" "testing" "time" @@ -54,6 +55,33 @@ func TestWriteErrorResponseDrainsRequestBodyBeforeWriting(t *testing.T) { assert.True(t, rr.readDeadlines[1].IsZero()) } +func TestWriteResponseOmitsBodyWhenNotAllowed(t *testing.T) { + payload := []byte("NoSuchKey") + for _, tc := range []struct { + name string + method string + status int + wantContentLength string + }{ + {"head request", http.MethodHead, http.StatusNotFound, strconv.Itoa(len(payload))}, + {"no content", http.MethodGet, http.StatusNoContent, ""}, + {"not modified", http.MethodGet, http.StatusNotModified, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(tc.method, "/bucket/object", nil) + rr := httptest.NewRecorder() + WriteResponse(rr, req, tc.status, payload, MimeXML) + assert.Empty(t, rr.Body.String()) + assert.Equal(t, tc.wantContentLength, rr.Header().Get("Content-Length")) + }) + } + + req := httptest.NewRequest(http.MethodGet, "/bucket/object", nil) + rr := httptest.NewRecorder() + WriteResponse(rr, req, http.StatusOK, payload, MimeXML) + assert.Equal(t, string(payload), rr.Body.String()) +} + func extractRequestIDFromBody(body string) string { re := regexp.MustCompile(`([^<]+)`) matches := re.FindStringSubmatch(body) diff --git a/weed/server/common.go b/weed/server/common.go index b45fe3a5c3..2749b4f3f7 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -109,7 +109,7 @@ func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj inter if httpStatus >= 400 { glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s", - r.Method, r.URL.String(), httpStatus, string(bytes)) + r.Method, redactedURL(r), httpStatus, string(bytes)) } w.Header().Set("Content-Type", "application/json") @@ -119,10 +119,21 @@ func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj inter return } +func redactedURL(r *http.Request) string { + if _, ok := r.URL.Query()["jwt"]; !ok { + return r.URL.String() + } + u := *r.URL + q := u.Query() + q.Set("jwt", "[redacted]") + u.RawQuery = q.Encode() + return u.String() +} + // wrapper for writeJson - just logs errors func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) { if err := writeJson(w, r, httpStatus, obj); err != nil { - glog.V(0).Infof("error writing JSON status %s %d: %v", r.URL, httpStatus, err) + glog.V(0).Infof("error writing JSON status %s %d: %v", redactedURL(r), httpStatus, err) glog.V(1).Infof("JSON content: %+v", obj) } } diff --git a/weed/server/common_test.go b/weed/server/common_test.go index ea81fabcad..caf960745f 100644 --- a/weed/server/common_test.go +++ b/weed/server/common_test.go @@ -400,3 +400,18 @@ func TestSubmitForClientHandlerFileSizeLimit(t *testing.T) { } }) } + +func TestRedactedURLDropsTheJwt(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/1,abc?jwt=secret-token&size=5", nil) + if got := redactedURL(r); strings.Contains(got, "secret-token") || !strings.Contains(got, "size=5") { + t.Errorf("got %q, want jwt redacted and other params kept", got) + } + r = httptest.NewRequest(http.MethodGet, "/1,abc?jwt=&jwt=secret-token&size=5", nil) + if got := redactedURL(r); strings.Contains(got, "secret-token") { + t.Errorf("got %q, want every jwt value redacted", got) + } + r = httptest.NewRequest(http.MethodGet, "/1,abc?size=5", nil) + if got := redactedURL(r); got != "/1,abc?size=5" { + t.Errorf("got %q, want the URL unchanged without a jwt", got) + } +} diff --git a/weed/server/filer_grpc_server_dlm.go b/weed/server/filer_grpc_server_dlm.go index 08427350a0..dc2864818d 100644 --- a/weed/server/filer_grpc_server_dlm.go +++ b/weed/server/filer_grpc_server_dlm.go @@ -32,7 +32,7 @@ func (fs *FilerServer) DistributedLock(ctx context.Context, req *filer_pb.LockRe if mErr := fs.checkMovedMarker(ctx, req.IsMoved, movedTo); mErr != nil { err = mErr } else if !req.IsMoved && movedTo != "" && movedTo != fs.option.Host { - glog.V(0).Infof("FILER LOCK: Forwarding to correct filer - from=%s to=%s", fs.option.Host, movedTo) + glog.V(3).Infof("FILER LOCK: Forwarding to correct filer - from=%s to=%s", fs.option.Host, movedTo) err = pb.WithFilerClient(false, 0, movedTo, fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { secondResp, err := client.DistributedLock(ctx, &filer_pb.LockRequest{ Name: req.Name, @@ -46,7 +46,9 @@ func (fs *FilerServer) DistributedLock(ctx context.Context, req *filer_pb.LockRe resp.LockOwner = secondResp.LockOwner resp.Error = secondResp.Error resp.Generation = secondResp.Generation - glog.V(0).Infof("FILER LOCK: Forwarded lock acquired - name=%s renewToken=%s", req.Name, resp.RenewToken) + if forwardedLockAcquired(secondResp) { + glog.V(3).Infof("FILER LOCK: Forwarded lock acquired - name=%s renewToken=%s", req.Name, resp.RenewToken) + } } else { glog.V(0).Infof("FILER LOCK: Forward failed - name=%s err=%v", req.Name, err) } @@ -70,6 +72,10 @@ func (fs *FilerServer) DistributedLock(ctx context.Context, req *filer_pb.LockRe return resp, nil } +func forwardedLockAcquired(resp *filer_pb.LockResponse) bool { + return resp != nil && resp.Error == "" && resp.RenewToken != "" +} + // Unlock is a grpc handler to handle FilerServer's UnlockRequest func (fs *FilerServer) DistributedUnlock(ctx context.Context, req *filer_pb.UnlockRequest) (resp *filer_pb.UnlockResponse, err error) { diff --git a/weed/server/filer_grpc_server_dlm_test.go b/weed/server/filer_grpc_server_dlm_test.go index 5713a9959e..7c82e9faac 100644 --- a/weed/server/filer_grpc_server_dlm_test.go +++ b/weed/server/filer_grpc_server_dlm_test.go @@ -34,3 +34,22 @@ func TestFindLockOwnerExpiredLockReturnsNotFound(t *testing.T) { require.True(t, ok) require.Equal(t, codes.NotFound, st.Code()) } + +func TestForwardedLockAcquired(t *testing.T) { + tests := []struct { + name string + resp *filer_pb.LockResponse + want bool + }{ + {name: "nil response"}, + {name: "empty token", resp: &filer_pb.LockResponse{}}, + {name: "application error", resp: &filer_pb.LockResponse{RenewToken: "token", Error: "lock already owned"}}, + {name: "acquired", resp: &filer_pb.LockResponse{RenewToken: "token"}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, forwardedLockAcquired(tt.resp)) + }) + } +} diff --git a/weed/server/filer_grpc_server_traverse_meta.go b/weed/server/filer_grpc_server_traverse_meta.go index c59d52d79d..7ad1014f81 100644 --- a/weed/server/filer_grpc_server_traverse_meta.go +++ b/weed/server/filer_grpc_server_traverse_meta.go @@ -2,6 +2,7 @@ package weed_server import ( "context" + "errors" "fmt" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -9,8 +10,16 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/viant/ptrie" + "google.golang.org/grpc/metadata" ) +// TraverseBfsMetadata streams every entry at and below req.Directory: by +// default a breadth-first walk, parents first, each response stamped with the +// path's commit timestamp. With request metadata TraverseModeHeaderKey=snapshot +// the entries come from one store snapshot instead (filer.EntryScanner), in any +// order, and the response header of the same name says which happened. Snapshot +// responses leave entry_ts_ns at 0: that lookup is one store round trip per +// entry, and the consumers that order by it use the walk. func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequest, stream filer_pb.SeaweedFiler_TraverseBfsMetadataServer) error { glog.V(0).Infof("TraverseBfsMetadata %v", req) @@ -40,12 +49,8 @@ func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequ // descend into. Queuing the entries themselves would hold the whole // subtree in memory, including every file's chunk list, and can exhaust // the filer on large trees (e.g. during a peer's first-time bootstrap). - sendEntry := func(entry *filer.Entry) error { + send := func(entry *filer.Entry, entryTsNs int64) error { parent, _ := entry.FullPath.DirAndName() - entryTsNs, err := fs.filer.CommitTsNs(ctx, entry.FullPath, entry.Mtime.UnixNano()) - if err != nil { - return fmt.Errorf("commit timestamp of %s: %w", entry.FullPath, err) - } if err := stream.Send(&filer_pb.TraverseBfsMetadataResponse{ Directory: parent, Entry: entry.ToProtoEntry(), @@ -55,6 +60,16 @@ func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequ } return nil } + sendEntry := func(entry *filer.Entry) error { + entryTsNs, err := fs.filer.CommitTsNs(ctx, entry.FullPath, entry.Mtime.UnixNano()) + if err != nil { + return fmt.Errorf("commit timestamp of %s: %w", entry.FullPath, err) + } + return send(entry, entryTsNs) + } + sendEntryUnstamped := func(entry *filer.Entry) error { + return send(entry, 0) + } dirEntry, err := fs.filer.FindEntry(ctx, util.FullPath(req.Directory)) if err != nil { @@ -63,6 +78,22 @@ func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequ if isExcluded(dirEntry.FullPath) { return nil } + + if traverseSnapshotRequested(ctx) { + if scanner, ok := fs.filer.Store.(filer.EntryScanner); ok && dirEntry.IsDirectory() { + served, scanErr := traverseFromSnapshot(ctx, scanner, dirEntry, isExcluded, stream.SendHeader, sendEntryUnstamped) + if scanErr != nil { + return scanErr + } + if served { + return nil + } + } + if err := stream.SendHeader(metadata.Pairs(filer_pb.TraverseModeHeaderKey, filer_pb.TraverseModeBfs)); err != nil { + return fmt.Errorf("send traverse mode header: %w", err) + } + } + if err := sendEntry(dirEntry); err != nil { return err } @@ -93,6 +124,62 @@ func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequ return nil } +// traverseSnapshotRequested reports whether the client asked for the entries +// from one store snapshot rather than a breadth-first walk. +func traverseSnapshotRequested(ctx context.Context) bool { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return false + } + modes := md.Get(filer_pb.TraverseModeHeaderKey) + return len(modes) > 0 && modes[0] == filer_pb.TraverseModeSnapshot +} + +// traverseFromSnapshot serves the traversal from the store's snapshot scan, +// root included: the scan yields the root's own row, so it is as current as +// its descendants (only "/" has no row and is sent as the synthetic root). +// It returns served=false, nothing sent, when the store declines before +// yielding anything; once the first entry is out, any failure is an error. +func traverseFromSnapshot(ctx context.Context, scanner filer.EntryScanner, dirEntry *filer.Entry, isExcluded func(util.FullPath) bool, sendHeader func(metadata.MD) error, sendEntry func(*filer.Entry) error) (served bool, err error) { + started := false + start := func() error { + if started { + return nil + } + started = true + if err := sendHeader(metadata.Pairs(filer_pb.TraverseModeHeaderKey, filer_pb.TraverseModeSnapshot)); err != nil { + return fmt.Errorf("send traverse mode header: %w", err) + } + if dirEntry.FullPath == "/" { + return sendEntry(dirEntry) + } + return nil + } + + scanErr := scanner.ScanEntries(ctx, dirEntry.FullPath, func(_ context.Context, entry *filer.Entry) error { + if isExcluded(entry.FullPath) { + return nil + } + if err := start(); err != nil { + return err + } + return sendEntry(entry) + }) + if scanErr != nil { + if !started && errors.Is(scanErr, filer.ErrUnsupportedEntryScan) { + glog.V(1).Infof("TraverseBfsMetadata %s: %v, walking instead", dirEntry.FullPath, scanErr) + return false, nil + } + glog.V(0).Infof("TraverseBfsMetadata %s: snapshot scan failed: %v", dirEntry.FullPath, scanErr) + return started, fmt.Errorf("snapshot scan of %s: %w", dirEntry.FullPath, scanErr) + } + // Nothing yielded (root gone since the lookup): still commit to the mode. + if err := start(); err != nil { + return true, err + } + return true, nil +} + func (fs *FilerServer) iterateDirectory(ctx context.Context, dirPath util.FullPath, fn func(entry *filer.Entry) error) (err error) { var lastFileName string var listErr error diff --git a/weed/server/filer_grpc_server_traverse_snapshot_test.go b/weed/server/filer_grpc_server_traverse_snapshot_test.go new file mode 100644 index 0000000000..14eb6fdfcc --- /dev/null +++ b/weed/server/filer_grpc_server_traverse_snapshot_test.go @@ -0,0 +1,170 @@ +package weed_server + +import ( + "context" + "sort" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/metadata" +) + +// ScanEntries makes the rename test store an EntryScanner: every entry at and +// below dirPath, straight from the map, like a store reading one snapshot. +func (s *renameTestStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error { + s.mu.Lock() + var paths []string + for p := range s.entries { + if p == "/" { + continue + } + if dirPath == "/" || p == string(dirPath) || len(p) > len(dirPath) && p[:len(dirPath)+1] == string(dirPath)+"/" { + paths = append(paths, p) + } + } + s.mu.Unlock() + // Deliberately not top-down: children before their parents. + sort.Sort(sort.Reverse(sort.StringSlice(paths))) + for _, p := range paths { + s.mu.Lock() + e := s.entries[p] + s.mu.Unlock() + if err := fn(ctx, e); err != nil { + return err + } + } + return nil +} + +var _ filer.EntryScanner = (*renameTestStore)(nil) + +// headerTraverseStream records the response header along with the entries. +type headerTraverseStream struct { + captureTraverseStream + headers []metadata.MD +} + +func (s *headerTraverseStream) SendHeader(md metadata.MD) error { + s.headers = append(s.headers, md) + return nil +} + +func snapshotRequestCtx() context.Context { + return metadata.NewIncomingContext(context.Background(), metadata.Pairs(filer_pb.TraverseModeHeaderKey, filer_pb.TraverseModeSnapshot)) +} + +func newTraverseTestStore() *renameTestStore { + store := newRenameTestStore() + for _, dir := range []string{"/", "/a", "/a/sub", "/b", "/.system"} { + store.entries[dir] = newDirectoryEntry(dir, 1) + } + for _, file := range []string{"/a/f1", "/a/sub/f2", "/b/f3", "/.system/secret"} { + store.entries[file] = newFileEntry(file, 2) + } + return store +} + +func TestTraverseBfsMetadataSnapshotMode(t *testing.T) { + store := newTraverseTestStore() + server := &FilerServer{filer: newRenameTestFiler(t, store)} + stream := &headerTraverseStream{captureTraverseStream: captureTraverseStream{ctx: snapshotRequestCtx()}} + + err := server.TraverseBfsMetadata(&filer_pb.TraverseBfsMetadataRequest{ + Directory: "/", + ExcludedPrefixes: []string{"/.system"}, + }, stream) + assert.NoError(t, err) + + if assert.Len(t, stream.headers, 1) { + assert.Equal(t, []string{filer_pb.TraverseModeSnapshot}, stream.headers[0].Get(filer_pb.TraverseModeHeaderKey)) + } + // The root comes first so a consumer knows what it asked for; everything + // else is complete but in the store's order, with the exclusion honored. + assert.Equal(t, "/", stream.visited[0]) + assert.ElementsMatch(t, []string{"/", "/a", "/a/f1", "/a/sub", "/a/sub/f2", "/b", "/b/f3"}, stream.visited) + + // No per-entry commit timestamp lookup in snapshot mode: entry_ts_ns is + // left "unrecorded" (0), which consumers already treat as "use the mtime". + for path, tsNs := range stream.entryTsNs { + assert.Zerof(t, tsNs, "entry_ts_ns for %s", path) + } +} + +// Without the request metadata the walk is unchanged: no header, parents first. +func TestTraverseBfsMetadataDefaultStillWalks(t *testing.T) { + store := newTraverseTestStore() + server := &FilerServer{filer: newRenameTestFiler(t, store)} + stream := &headerTraverseStream{captureTraverseStream: captureTraverseStream{ctx: context.Background()}} + + err := server.TraverseBfsMetadata(&filer_pb.TraverseBfsMetadataRequest{Directory: "/"}, stream) + assert.NoError(t, err) + assert.Empty(t, stream.headers) + seen := make(map[string]bool) + for _, path := range stream.visited { + if path != "/" { + parent, _ := util.FullPath(path).DirAndName() + assert.Truef(t, seen[parent], "parent %s of %s not streamed first", parent, path) + } + seen[path] = true + } +} + +// A store that cannot scan (here: path-specific stores are configured, so one +// snapshot per store is not one snapshot) makes the filer walk and say so. +func TestTraverseBfsMetadataSnapshotRequestedButUnsupported(t *testing.T) { + store := newTraverseTestStore() + f := newRenameTestFiler(t, store) + f.Store.AddPathSpecificStore("/elsewhere", "elsewhere", newRenameTestStore()) + server := &FilerServer{filer: f} + stream := &headerTraverseStream{captureTraverseStream: captureTraverseStream{ctx: snapshotRequestCtx()}} + + err := server.TraverseBfsMetadata(&filer_pb.TraverseBfsMetadataRequest{Directory: "/a"}, stream) + assert.NoError(t, err) + if assert.Len(t, stream.headers, 1) { + assert.Equal(t, []string{filer_pb.TraverseModeBfs}, stream.headers[0].Get(filer_pb.TraverseModeHeaderKey)) + } + assert.Equal(t, []string{"/a", "/a/f1", "/a/sub", "/a/sub/f2"}, stream.visited) +} + +type fakeScanner struct { + entries []*filer.Entry + err error // returned after the entries +} + +func (s *fakeScanner) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error { + for _, e := range s.entries { + if err := fn(ctx, e); err != nil { + return err + } + } + return s.err +} + +type traverseRecorder struct { + headers []metadata.MD + sent []string +} + +func (r *traverseRecorder) sendHeader(md metadata.MD) error { + r.headers = append(r.headers, md) + return nil +} +func (r *traverseRecorder) sendEntry(e *filer.Entry) error { + r.sent = append(r.sent, string(e.FullPath)) + return nil +} + +func noExclusion(util.FullPath) bool { return false } + +// A store that only decides it cannot scan after yielding rows must not make +// the stream fall back to a walk: entries are already out under the snapshot header. +func TestTraverseFromSnapshotLateUnsupportedIsAnError(t *testing.T) { + rec := &traverseRecorder{} + scanner := &fakeScanner{entries: []*filer.Entry{newFileEntry("/ckpt/x", 2)}, err: filer.ErrUnsupportedEntryScan} + served, err := traverseFromSnapshot(context.Background(), scanner, newDirectoryEntry("/ckpt", 1), noExclusion, rec.sendHeader, rec.sendEntry) + assert.Error(t, err) + assert.True(t, served) +} diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 0864409c3c..177093c501 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -122,6 +123,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte return } + if so.Fsync { + ctx = filer.WithSyncWrite(ctx) + } + if util.FullPath(r.URL.Path).IsLongerFileName(so.MaxFileNameLength) { glog.V(1).InfolnCtx(ctx, "post", r.RequestURI, ": ", "entry name too long") w.WriteHeader(http.StatusRequestURITooLong) diff --git a/weed/server/volume_grpc_tier_upload.go b/weed/server/volume_grpc_tier_upload.go index 540dd383d0..cd60cae528 100644 --- a/weed/server/volume_grpc_tier_upload.go +++ b/weed/server/volume_grpc_tier_upload.go @@ -5,6 +5,7 @@ import ( "os" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" @@ -92,8 +93,9 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.Name(), err) } - // save the remote file to volume tier info - v.GetVolumeInfo().Files = append(v.GetVolumeInfo().GetFiles(), &volume_server_pb.RemoteFile{ + // record the remote file and switch to it, unless a compaction commit + // replaced the .dat while it was being copied + if err := v.AttachRemoteDatFile(diskFile, &volume_server_pb.RemoteFile{ BackendType: backendType, BackendId: backendId, Key: key, @@ -101,14 +103,20 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi FileSize: uint64(size), ModifiedTime: uint64(modTime.Unix()), Extension: ".dat", - }) - - if err := v.SaveVolumeInfo(); err != nil { - return fmt.Errorf("volume %d failed to save remote file info: %v", v.Id, err) - } - - if err := v.LoadRemoteFile(); err != nil { - return fmt.Errorf("volume %d failed to load remote file: %v", v.Id, err) + }); err != nil { + // Remove the copy unless the .vif already names it. + recorded := false + for _, f := range v.GetVolumeInfo().GetFiles() { + if f.Key == key && f.BackendType == backendType && f.BackendId == backendId { + recorded = true + } + } + if !recorded { + if delErr := backendStorage.DeleteFile(key); delErr != nil { + glog.Warningf("volume %d: remove unused remote copy %s: %v", v.Id, key, delErr) + } + } + return fmt.Errorf("volume %d failed to attach remote file: %v", v.Id, err) } // Deletes must reach the index now, not only after the next restart. v.MarkTieredDeletable() diff --git a/weed/server/volume_grpc_tier_upload_compact_test.go b/weed/server/volume_grpc_tier_upload_compact_test.go new file mode 100644 index 0000000000..9295ed81e6 --- /dev/null +++ b/weed/server/volume_grpc_tier_upload_compact_test.go @@ -0,0 +1,107 @@ +package weed_server + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// afterCopyBackend runs a hook once the .dat has been copied, before the +// upload handler records the remote file. +type afterCopyBackend struct { + tierTimestampTestBackend + afterCopy func() +} + +func (b *afterCopyBackend) CopyFile(file *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (string, int64, error) { + key, size, err := b.tierTimestampTestBackend.CopyFile(file, fn, concurrency) + if err == nil && b.afterCopy != nil { + b.afterCopy() + } + return key, size, err +} + +// A compaction committed after the tier upload copied the .dat, but before the +// upload switched the volume to the remote object, leaves that object with the +// old layout and the volume with the compacted index. The upload must not +// attach it. +func TestTierUploadRefusesADatReplacedByACompactionCommit(t *testing.T) { + root := t.TempDir() + b := &afterCopyBackend{tierTimestampTestBackend: tierTimestampTestBackend{root: root}} + backend.BackendStorages[tierTimestampTestBackendName] = b + t.Cleanup(func() { delete(backend.BackendStorages, tierTimestampTestBackendName) }) + + store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{t.TempDir()}, []int32{10}, []util.MinFreeSpace{{}}, "", storage.NeedleMapInMemory, + []types.DiskType{types.HardDriveType}, nil, 0, stats.DefaultDiskIOProbeConfig()) + t.Cleanup(store.Close) + if err := store.AddVolume(1, "", storage.NeedleMapInMemory, "000", "", 0, needle.Version3, 0, types.HardDriveType, 0); err != nil { + t.Fatalf("add volume: %v", err) + } + written := map[uint64][]byte{} + for i := uint64(1); i <= 10; i++ { + n := &needle.Needle{Id: types.Uint64ToNeedleId(i), Cookie: 7, Data: bytes.Repeat([]byte{byte(i)}, 1024)} + n.Checksum = needle.NewCRC(n.Data) + if _, err := store.WriteVolumeNeedle(1, n, false, false); err != nil { + t.Fatalf("write %d: %v", i, err) + } + written[i] = n.Data + } + for i := uint64(1); i <= 5; i++ { + if _, err := store.DeleteVolumeNeedle(1, &needle.Needle{Id: types.Uint64ToNeedleId(i)}); err != nil { + t.Fatalf("delete %d: %v", i, err) + } + } + v := store.GetVolume(1) + if err := v.CompactByIndex(nil); err != nil { + t.Fatalf("compact: %v", err) + } + if err := store.MarkVolumeReadonly(1, false, false); err != nil { + t.Fatalf("mark read-only: %v", err) + } + + committed := false + b.afterCopy = func() { + if err := v.CommitCompact(); err != nil { + t.Errorf("commit: %v", err) + return + } + committed = true + } + vs := &VolumeServer{store: store} + err := vs.VolumeTierMoveDatToRemote(&volume_server_pb.VolumeTierMoveDatToRemoteRequest{ + VolumeId: 1, DestinationBackendName: tierTimestampTestBackendName, + }, &discardServerStream[volume_server_pb.VolumeTierMoveDatToRemoteResponse]{}) + if !committed { + t.Fatal("the compaction did not commit between the copy and the attach") // positive control + } + if err == nil { + t.Errorf("the upload attached a remote copy of a .dat that a compaction had replaced") + } + if v.HasRemoteFile() { + t.Errorf("the volume is tiered to the stale remote copy") + } + if entries, _ := os.ReadDir(root); len(entries) != 0 { + t.Errorf("the abandoned remote copy was left behind: %v", entries) + } + if !util.FileExists(filepath.Join(v.FileName(".dat"))) { + t.Fatalf("the local .dat is gone") + } + for i := uint64(6); i <= 10; i++ { + n := &needle.Needle{Id: types.Uint64ToNeedleId(i), Cookie: 7} + if _, err := store.ReadVolumeNeedle(1, n, nil, nil); err != nil { + t.Errorf("needle %d does not read after the upload: %v", i, err) + } else if !bytes.Equal(n.Data, written[i]) { + t.Errorf("needle %d reads different data", i) + } + } +} diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 957f9a17e6..aab0245f5e 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -40,10 +40,20 @@ func NotFound(w http.ResponseWriter) { w.WriteHeader(http.StatusNotFound) } -func InternalError(w http.ResponseWriter) { +func InternalError(w http.ResponseWriter, r *http.Request, err error) { stats.VolumeServerFileReadFailures.Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetInternal).Inc() - w.WriteHeader(http.StatusInternalServerError) + if err == nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if r.Method == http.MethodHead { + glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and error:%v", + r.Method, redactedURL(r), http.StatusInternalServerError, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + writeJsonError(w, r, http.StatusInternalServerError, err) } func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Request) { @@ -92,8 +102,8 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re r.URL.RawQuery = query.Encode() request, err := http.NewRequest(http.MethodGet, r.URL.String(), nil) if err != nil { - glog.V(0).Infof("failed to instance http request of url %s: %v", r.URL.String(), err) - InternalError(w) + glog.V(0).Infof("failed to instance http request of url %s: %v", redactedURL(r), err) + InternalError(w, r, nil) return } for k, vv := range r.Header { @@ -105,8 +115,8 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re response, err := util_http.GetGlobalHttpClient().Do(request) if err != nil { stats.VolumeServerHandlerCounter.WithLabelValues(stats.FailedReadProxyReq).Inc() - glog.V(0).Infof("request remote url %s: %v", r.URL.String(), err) - InternalError(w) + glog.V(0).Infof("request remote url %s: %v", redactedURL(r), err) + InternalError(w, r, nil) return } defer util_http.CloseResponse(response) @@ -217,7 +227,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if invalid_needle || err == storage.ErrorDeleted { NotFound(w) } else { - InternalError(w) + InternalError(w, r, err) } return } diff --git a/weed/server/volume_server_handlers_read_test.go b/weed/server/volume_server_handlers_read_test.go index 83f7e3bc55..758ff52ea8 100644 --- a/weed/server/volume_server_handlers_read_test.go +++ b/weed/server/volume_server_handlers_read_test.go @@ -1,11 +1,37 @@ package weed_server import ( + "errors" "net/http" "net/http/httptest" + "strings" "testing" ) +func TestInternalErrorWritesTheReadError(t *testing.T) { + readErr := errors.New("invalid CRC for needle 2a (got 0a1b2c3d, want 4e5f6071), data on disk corrupted: needle data corrupted") + w := httptest.NewRecorder() + InternalError(w, httptest.NewRequest(http.MethodGet, "/1,abc", nil), readErr) + if w.Code != http.StatusInternalServerError { + t.Fatalf("status %d, want 500", w.Code) + } + if body := strings.TrimSpace(w.Body.String()); body != `{"error":"`+readErr.Error()+`"}` { + t.Fatalf("body %q", body) + } + + w = httptest.NewRecorder() + InternalError(w, httptest.NewRequest(http.MethodGet, "/1,abc", nil), nil) + if w.Code != http.StatusInternalServerError || w.Body.Len() != 0 { + t.Fatalf("without an error: status %d body %q, want a bare 500", w.Code, w.Body.String()) + } + + w = httptest.NewRecorder() + InternalError(w, httptest.NewRequest(http.MethodHead, "/1,abc", nil), readErr) + if w.Code != http.StatusInternalServerError || w.Body.Len() != 0 { + t.Fatalf("on a HEAD request: status %d body %q, want a bare 500", w.Code, w.Body.String()) + } +} + // A HEAD only attempts a metadata-only read, like a GET: a needle up to // PagedReadLimit is read whole. Demanding metadata only would turn that one // read into a header read followed by a trailer read. diff --git a/weed/sftpd/auth/password.go b/weed/sftpd/auth/password.go index 21216d3ff8..7498adc7e0 100644 --- a/weed/sftpd/auth/password.go +++ b/weed/sftpd/auth/password.go @@ -37,8 +37,9 @@ func (a *PasswordAuthenticator) Authenticate(conn ssh.ConnMetadata, password []b return nil, fmt.Errorf("password authentication disabled") } - // Validate password against user store - if a.userStore.ValidatePassword(username, password) { + // An empty password is never valid; skip the store so a store without its + // own empty-secret check cannot match on zero-length input. + if len(password) != 0 && a.userStore.ValidatePassword(username, password) { return &ssh.Permissions{ Extensions: map[string]string{ "username": username, diff --git a/weed/sftpd/auth/password_test.go b/weed/sftpd/auth/password_test.go new file mode 100644 index 0000000000..b54f6d29ce --- /dev/null +++ b/weed/sftpd/auth/password_test.go @@ -0,0 +1,43 @@ +package auth + +import "testing" + +// acceptAllStore records whether the store was consulted and accepts any password. +type acceptAllStore struct { + stubStore + called bool +} + +func (s *acceptAllStore) ValidatePassword(string, []byte) bool { + s.called = true + return true +} + +// An empty password must be rejected before the store is consulted, so a store +// without its own empty-secret check cannot authenticate on zero-length input. +func TestPasswordAuthenticatorRejectsEmptyPassword(t *testing.T) { + store := &acceptAllStore{} + a := NewPasswordAuthenticator(store, true) + + for _, password := range [][]byte{nil, {}} { + if _, err := a.Authenticate(&fakeConnMetadata{user: "alice"}, password); err == nil { + t.Fatalf("expected empty password %v to be rejected", password) + } + } + if store.called { + t.Fatal("store must not be consulted for an empty password") + } +} + +func TestPasswordAuthenticatorConsultsStore(t *testing.T) { + store := &acceptAllStore{} + a := NewPasswordAuthenticator(store, true) + + perms, err := a.Authenticate(&fakeConnMetadata{user: "alice"}, []byte("s3cret")) + if err != nil { + t.Fatalf("expected non-empty password to reach the store: %v", err) + } + if perms.Extensions["username"] != "alice" { + t.Fatalf("expected username extension alice, got %q", perms.Extensions["username"]) + } +} diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index da88bf34e7..b936252f62 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -20,6 +20,9 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) func init() { @@ -152,6 +155,133 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. // log; volume.fsck and fs.verify must leave it false so the live log-file // chunks are accounted for — otherwise fsck reports them as orphans and could // delete referenced needles. +// doTraverseAndSaving feeds every entry at and below path through genFn and +// saveFn: from one snapshot of the filer store when the filer can provide it +// (so a file renamed during the traversal is not missed), by walking otherwise. +func doTraverseAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error { + if used, err := doTraverseSnapshotAndSaving(filerClient, writer, path, verbose, skipSystemLog, genFn, saveFn); used || err != nil { + return err + } + return doTraverseBfsAndSaving(filerClient, writer, path, verbose, skipSystemLog, genFn, saveFn) +} + +// doTraverseSnapshotAndSaving asks the filer to traverse path itself and +// consumes the stream only when the filer serves it from a store snapshot +// (filer_pb.TraverseModeHeaderKey). Otherwise it returns used=false having +// read nothing, and the caller walks as before. +func doTraverseSnapshotAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) (used bool, err error) { + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var excludedPrefixes []string + if skipSystemLog { + excludedPrefixes = append(excludedPrefixes, filer.SystemLogDir) + } + + // The walk tolerates a path that does not exist (it saves nothing); the + // filer-side traversal reports it as an error, so let the walk have it. + // Any other failure must stop here: a walk that could then reach the filer + // reads renamed-away entries as absent, and fsck would purge their chunks. + if _, _, _, getErr := filer_pb.GetEntry(ctx, filerClient, util.FullPath(path)); getErr != nil { + if errors.Is(getErr, filer_pb.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("traverse %s: %w", path, getErr) + } + + err = filerClient.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error { + ctx := metadata.AppendToOutgoingContext(ctx, filer_pb.TraverseModeHeaderKey, filer_pb.TraverseModeSnapshot) + stream, err := client.TraverseBfsMetadata(ctx, &filer_pb.TraverseBfsMetadataRequest{ + Directory: path, + ExcludedPrefixes: excludedPrefixes, + }) + if err != nil { + return err + } + header, err := stream.Header() + if err != nil { + if status.Code(err) == codes.Unimplemented { + return nil // a filer without the RPC at all: walk + } + return fmt.Errorf("traverse %s: %w", path, err) + } + if modes := header.Get(filer_pb.TraverseModeHeaderKey); len(modes) == 0 || modes[0] != filer_pb.TraverseModeSnapshot { + // An older filer (no header), a store that cannot scan ("bfs"), or + // a stream that failed before any header, which Header() reports + // as no header and nil: only Recv tells, and a failure is no + // licence to walk instead. + if _, recvErr := stream.Recv(); recvErr != nil && !errors.Is(recvErr, io.EOF) && status.Code(recvErr) != codes.Unimplemented { + return fmt.Errorf("traverse %s: %w", path, recvErr) + } + return nil + } + used = true + + var wg sync.WaitGroup + wg.Add(1) + outputChan := make(chan interface{}, 1024) + saveErrChan := make(chan error, 1) + go func() { + saveErr := saveFn(outputChan) + saveErrChan <- saveErr + if saveErr != nil { + cancel() + } + wg.Done() + }() + + var dirCount, fileCount uint64 + var recvErr error + for { + resp, err := stream.Recv() + if err != nil { + if !errors.Is(err, io.EOF) { + recvErr = fmt.Errorf("traverse %s: %w", path, err) + } + break + } + if resp.Entry == nil { + continue + } + protoMessage := &filer_pb.FullEntry{ + Dir: resp.Directory, + Entry: resp.Entry, + } + if genErr := genFn(ctx, protoMessage, outputChan); genErr != nil { + recvErr = genErr + cancel() + break + } + if resp.Entry.IsDirectory { + dirCount++ + } else { + fileCount++ + } + if verbose { + println(util.NewFullPath(resp.Directory, resp.Entry.Name)) + } + } + + close(outputChan) + wg.Wait() + saveErr := <-saveErrChan + + if recvErr != nil { + return recvErr + } + if saveErr != nil { + return saveErr + } + if writer != nil { + fmt.Fprintf(writer, "collected filer entries from a consistent store snapshot\n") + fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount) + } + return nil + }) + return used, err +} + func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error { ctx, cancel := context.WithCancel(context.Background()) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 63981282b6..7578c71823 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -63,6 +63,7 @@ type commandVolumeFsck struct { tempFolder string verbose *bool forcePurging *bool + purgeConcurrency *int skipEcVolumes *bool findMissingChunksInFiler *bool verifyNeedle *bool @@ -99,6 +100,12 @@ func (c *commandVolumeFsck) Help() string { A newly uploaded chunk may appear as orphan if metadata commit is still pending. The default 5h cutoff provides sufficient buffer for metadata commits. + Step 2 reads the filer from one consistent snapshot when its store supports + that (the SQL stores do), so a file renamed while the filer is being read is + still counted and its chunks are not mistaken for orphans. Other stores are + walked directory by directory; there, avoid running with -reallyDeleteFromVolume + while files are being renamed or moved. + ` } @@ -117,6 +124,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer. Currently this only works with default filerGroup.") c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") + c.purgeConcurrency = fsckCommand.Int("purgeConcurrency", 16, "parallel filer deletes for -reallyDeleteFilerEntries") tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") cutoffTimeAgo := fsckCommand.Duration("cutoffTimeAgo", 5*time.Hour, "only include entries on volume servers before this cutoff time to check orphan chunks") modifyTimeAgo := fsckCommand.Duration("modifyTimeAgo", 0, "only include entries after this modify time to check orphan chunks") @@ -148,7 +156,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. } } c.env = commandEnv - c.writer = writer + c.writer = &lockedWriter{w: writer} if c.collectionMatcher, err = wildcard.CompileCollectionMatcher(*c.collection); err != nil { return fmt.Errorf("invalid collection pattern %q: %w", *c.collection, err) @@ -322,7 +330,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m } }() - return doTraverseBfsAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, false, + return doTraverseAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, false, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if *c.verbose && entry.Entry.IsDirectory { fmt.Fprintf(c.writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name)) @@ -360,10 +368,11 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m } select { case outputChan <- &Item{ - vid: chunk.Fid.VolumeId, - fileKey: chunk.Fid.FileKey, - cookie: chunk.Fid.Cookie, - path: util.NewFullPath(entry.Dir, entry.Entry.Name), + vid: chunk.Fid.VolumeId, + fileKey: chunk.Fid.FileKey, + cookie: chunk.Fid.Cookie, + path: util.NewFullPath(entry.Dir, entry.Entry.Name), + hardLinkId: entry.Entry.GetHardLinkId(), }: case <-ctx.Done(): return ctx.Err() @@ -373,6 +382,12 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m }, func(outputChan chan interface{}) error { buffer := make([]byte, readbufferSize) + var queuePurge func(purgeItem) + var finishPurge func() + if purgeAbsent { + queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete) + defer finishPurge() + } for item := range outputChan { i := item.(*Item) if f, ok := files[i.vid]; ok { @@ -400,7 +415,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m fmt.Fprintf(c.writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) if purgeAbsent { fmt.Fprintf(c.writer, "deleting path %s after volume not found\n", i.path) - c.httpDelete(i.path) + queuePurge(purgeItem{path: i.path, hardLinkId: i.hardLinkId}) } } } @@ -615,10 +630,24 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId } type Item struct { - vid uint32 - fileKey uint64 - cookie uint32 - path util.FullPath + vid uint32 + fileKey uint64 + cookie uint32 + path util.FullPath + hardLinkId []byte +} + +// lockedWriter serializes concurrent writers — the traversal, the consumer, +// and the purge workers all log through c.writer at the same time. +type lockedWriter struct { + mu sync.Mutex + w io.Writer +} + +func (lw *lockedWriter) Write(p []byte) (int, error) { + lw.mu.Lock() + defer lw.mu.Unlock() + return lw.w.Write(p) } func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath)) error { @@ -686,6 +715,47 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, vo return nil } +type purgeItem struct { + path util.FullPath + hardLinkId []byte +} + +// startPurgeWorkers fans deletes across at least one worker and returns a +// queue function plus a finish function that closes the channels and waits +// for in-flight deletes. Paths that share a hard link id always land on the +// same worker, so their shared hard-link counter updates stay serialized — +// concurrent deletes of the same id could both read count N and write N-1, +// leaving the shared record behind after every path is gone. +func startPurgeWorkers(workers int, deleteFn func(util.FullPath)) (queue func(purgeItem), finish func()) { + if workers < 1 { + workers = 1 + } + chans := make([]chan purgeItem, workers) + var wg sync.WaitGroup + for w := range chans { + chans[w] = make(chan purgeItem, 128) + wg.Add(1) + go func(ch <-chan purgeItem) { + defer wg.Done() + for it := range ch { + deleteFn(it.path) + } + }(chans[w]) + } + return func(it purgeItem) { + key := []byte(it.path) + if len(it.hardLinkId) > 0 { + key = it.hardLinkId + } + chans[int(uint32(util.HashToInt32(key))%uint32(workers))] <- it + }, func() { + for _, ch := range chans { + close(ch) + } + wg.Wait() + } +} + func (c *commandVolumeFsck) httpDelete(path util.FullPath) { req, err := http.NewRequest(http.MethodDelete, "", nil) if err != nil { diff --git a/weed/shell/command_volume_fsck_test.go b/weed/shell/command_volume_fsck_test.go index 056e25b52c..50bdd0f8fa 100644 --- a/weed/shell/command_volume_fsck_test.go +++ b/weed/shell/command_volume_fsck_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "sort" "strings" + "sync/atomic" "testing" "time" @@ -226,3 +227,86 @@ func TestVolumeFsckPurgeEmptyDirectoriesKeepsChangedDirectory(t *testing.T) { t.Errorf("deleted %v, expected a directory written to after the lookup to be kept", stub.deleted) } } + +// The purge pool may run at most `workers` deletes at once, and at least one +// even when the flag is set to zero. +func TestVolumeFsckPurgeWorkerLimit(t *testing.T) { + gate := make(chan struct{}) + var inFlight, maxSeen, processed atomic.Int32 + queue, finish := startPurgeWorkers(3, func(util.FullPath) { + n := inFlight.Add(1) + for { + m := maxSeen.Load() + if n <= m || maxSeen.CompareAndSwap(m, n) { + break + } + } + <-gate + inFlight.Add(-1) + processed.Add(1) + }) + + for i := 0; i < 10; i++ { + queue(purgeItem{path: util.FullPath(fmt.Sprintf("/p/%d", i))}) + } + deadline := time.Now().Add(5 * time.Second) + for inFlight.Load() < 3 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := inFlight.Load(); got != 3 { + t.Fatalf("in-flight deletes %d, want the 3-worker limit", got) + } + time.Sleep(10 * time.Millisecond) + if got := maxSeen.Load(); got > 3 { + t.Fatalf("saw %d concurrent deletes, want at most 3", got) + } + close(gate) + finish() + if got := processed.Load(); got != 10 { + t.Fatalf("processed %d deletes, want all 10 drained", got) + } +} + +func TestVolumeFsckPurgeWorkersAtLeastOne(t *testing.T) { + var processed atomic.Int32 + queue, finish := startPurgeWorkers(0, func(util.FullPath) { processed.Add(1) }) + for i := 0; i < 5; i++ { + queue(purgeItem{path: util.FullPath(fmt.Sprintf("/p/%d", i))}) + } + finish() + if got := processed.Load(); got != 5 { + t.Fatalf("processed %d deletes, want 5", got) + } +} + +// Deletes of paths sharing a hard link id must never overlap: the filer +// decrements a shared counter non-atomically, so concurrent deletes of the +// same id would both read N and write back N-1. +func TestVolumeFsckPurgeHardLinksSerialized(t *testing.T) { + var inFlight, maxSeen, processed atomic.Int32 + gate := make(chan struct{}) + queue, finish := startPurgeWorkers(8, func(util.FullPath) { + n := inFlight.Add(1) + for { + m := maxSeen.Load() + if n <= m || maxSeen.CompareAndSwap(m, n) { + break + } + } + <-gate + inFlight.Add(-1) + processed.Add(1) + }) + for i := 0; i < 8; i++ { + queue(purgeItem{path: util.FullPath(fmt.Sprintf("/hl/%d", i)), hardLinkId: []byte{9, 9, 9}}) + } + time.Sleep(50 * time.Millisecond) + if got := maxSeen.Load(); got > 1 { + t.Fatalf("saw %d concurrent deletes on one hard link id, want 1", got) + } + close(gate) + finish() + if got := processed.Load(); got != 8 { + t.Fatalf("processed %d deletes, want 8", got) + } +} diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 315dbcb585..3bbb363b18 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -26,12 +26,13 @@ type ShellOptions struct { Masters *string GrpcDialOption grpc.DialOption // shell transient context - FilerHost string - FilerPort int64 - FilerGroup *string - FilerAddress pb.ServerAddress - Directory string - Debug bool + FilerHost string + FilerPort int64 + FilerGroup *string + FilerAddress pb.ServerAddress + Directory string + Debug bool + ConnectTimeout time.Duration } type CommandEnv struct { diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index c3e0d86d7f..6aae477b0f 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -10,6 +10,7 @@ import ( "path" "slices" "strings" + "time" "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -57,14 +58,28 @@ func RunShell(options ShellOptions) error { commandEnv := NewCommandEnv(&options) + connectTimeout := options.ConnectTimeout + if connectTimeout <= 0 { + connectTimeout = 30 * time.Second + } ctx := context.Background() - go commandEnv.MasterClient.KeepConnectedToMaster(ctx) - commandEnv.MasterClient.WaitUntilConnected(ctx) + connectCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + reconnectCtx, cancelReconnect := context.WithCancel(ctx) + defer cancelReconnect() + go commandEnv.MasterClient.KeepConnectedToMaster(reconnectCtx) + if commandEnv.MasterClient.GetMaster(connectCtx) == "" { + err := fmt.Errorf("cannot connect to master %s within %s", *options.Masters, connectTimeout) + fmt.Fprintln(os.Stderr, err) + return err + } if commandEnv.option.FilerAddress == "" { var filers []pb.ServerAddress - commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { - resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{ + discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout) + defer cancelDiscover() + listErr := commandEnv.MasterClient.WithClient(discoverCtx, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(discoverCtx, &master_pb.ListClusterNodesRequest{ ClientType: cluster.FilerType, FilerGroup: *options.FilerGroup, }) @@ -79,6 +94,8 @@ func RunShell(options ShellOptions) error { }) if len(filers) > 0 { commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))] + } else if listErr != nil { + fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr) } if options.Debug { if len(filers) > 0 { diff --git a/weed/shell/shell_liner_test.go b/weed/shell/shell_liner_test.go index 6d9bda9e7c..21a707ac8a 100644 --- a/weed/shell/shell_liner_test.go +++ b/weed/shell/shell_liner_test.go @@ -5,6 +5,7 @@ import ( "io" "strings" "testing" + "time" ) type fakeCommand struct { @@ -44,6 +45,24 @@ func TestProcessEachCmdReturnsErrors(t *testing.T) { } } +func TestRunShellConnectTimeout(t *testing.T) { + masters := "127.0.0.1:1" + filerGroup := "" + + start := time.Now() + err := RunShell(ShellOptions{ + Masters: &masters, + FilerGroup: &filerGroup, + ConnectTimeout: 200 * time.Millisecond, + }) + if err == nil || !strings.Contains(err.Error(), "cannot connect to master") { + t.Fatalf("expected a master connection timeout, got: %v", err) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Fatalf("RunShell should give up near the connect timeout, took %v", elapsed) + } +} + func TestProcessEachCmdPreservesRegisteredCommandError(t *testing.T) { doErr := errors.New("daily_run: shard=3: recovery walk failed") failing := &fakeCommand{name: "test.failing.command", err: doErr} diff --git a/weed/stats/disk.go b/weed/stats/disk.go index a75cbbab41..67bc06fb86 100644 --- a/weed/stats/disk.go +++ b/weed/stats/disk.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "google.golang.org/protobuf/proto" ) type DiskIOProbeConfig struct { @@ -91,7 +92,7 @@ type diskState struct { statFailureCount int statSuccessCount int lastStatErr error - lastGoodStatus volume_server_pb.DiskStatus + lastGoodStatus *volume_server_pb.DiskStatus hasLastGood bool samples []ioSample @@ -497,7 +498,7 @@ func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config } func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) { - s.lastGoodStatus = *disk + s.lastGoodStatus = proto.Clone(disk).(*volume_server_pb.DiskStatus) s.hasLastGood = true } @@ -508,7 +509,8 @@ func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) dir := disk.Dir errorMessage := disk.Error - *disk = s.lastGoodStatus + proto.Reset(disk) + proto.Merge(disk, s.lastGoodStatus) disk.Dir = dir disk.Error = errorMessage } diff --git a/weed/storage/erasure_coding/distribution/README.md b/weed/storage/erasure_coding/distribution/README.md index 91cfd8fe8a..dbd2dfe096 100644 --- a/weed/storage/erasure_coding/distribution/README.md +++ b/weed/storage/erasure_coding/distribution/README.md @@ -30,8 +30,8 @@ fmt.Println(dist.Summary()) // Replication: replication=110 (DCs:2, Racks/DC:2, Nodes/Rack:1) // Distribution Plan: // Data Centers: 2 (target 7 shards each, max 9) -// Racks per DC: 2 (target 4 shards each, max 6) -// Nodes per Rack: 1 (target 4 shards each, max 6) +// Racks per DC: 2 (target 4 shards each, max 4) +// Nodes per Rack: 1 (target 4 shards each, max 4) ``` ### Custom EC Ratios (seaweed-enterprise) diff --git a/weed/storage/erasure_coding/distribution/analysis.go b/weed/storage/erasure_coding/distribution/analysis.go index b939df53ee..e19d4624a0 100644 --- a/weed/storage/erasure_coding/distribution/analysis.go +++ b/weed/storage/erasure_coding/distribution/analysis.go @@ -1,5 +1,103 @@ package distribution +// NewTopologyAnalysis creates an empty topology analysis. +func NewTopologyAnalysis() *TopologyAnalysis { + return &TopologyAnalysis{ + ShardsByDC: make(map[string]int), + ShardsByRack: make(map[string]int), + ShardsByNode: make(map[string]int), + DCToShards: make(map[string][]int), + RackToShards: make(map[string][]int), + NodeToShards: make(map[string][]int), + DCToRacks: make(map[string][]string), + RackToNodes: make(map[string][]*TopologyNode), + AllNodes: make(map[string]*TopologyNode), + } +} + +// AddNode registers a node in the topology, including any shards it already +// holds for the volume being analyzed. A node id may only register once; +// registering it again updates its capacity and adds any new shards. The +// first registration wins for the node's place in the topology. +func (a *TopologyAnalysis) AddNode(node *TopologyNode) { + registered, ok := a.AllNodes[node.NodeID] + if !ok { + registered = node + a.AllNodes[node.NodeID] = node + a.RackToNodes[rackKey(node.DataCenter, node.Rack)] = append(a.RackToNodes[rackKey(node.DataCenter, node.Rack)], node) + if !contains(a.DCToRacks[node.DataCenter], node.Rack) { + a.DCToRacks[node.DataCenter] = append(a.DCToRacks[node.DataCenter], node.Rack) + } + } else { + registered.FreeSlots = node.FreeSlots + registered.TotalShards = node.TotalShards + } + seen := node.ShardIDs[:0] + for _, shardID := range node.ShardIDs { + if !contains(seen, shardID) { + seen = append(seen, shardID) + } + } + node.ShardIDs = seen + for _, shardID := range node.ShardIDs { + a.AddShardLocation(ShardLocation{ + ShardID: shardID, + NodeID: registered.NodeID, + DataCenter: registered.DataCenter, + Rack: registered.Rack, + }) + } +} + +// AddShardLocation records one shard's placement in the topology. Repeated +// calls for a shard already known to its node are ignored, so callers may +// either set ShardIDs on the TopologyNode or register each shard here. A +// location naming an unregistered node registers that node so the rebalancer +// can see the shard; for a registered node the node's own dc/rack win. +func (a *TopologyAnalysis) AddShardLocation(loc ShardLocation) { + node, ok := a.AllNodes[loc.NodeID] + if !ok { + node = &TopologyNode{NodeID: loc.NodeID, DataCenter: loc.DataCenter, Rack: loc.Rack} + a.AddNode(node) + } + if contains(a.NodeToShards[loc.NodeID], loc.ShardID) { + return + } + a.ShardsByDC[node.DataCenter]++ + a.ShardsByRack[rackKey(node.DataCenter, node.Rack)]++ + a.ShardsByNode[loc.NodeID]++ + a.DCToShards[node.DataCenter] = append(a.DCToShards[node.DataCenter], loc.ShardID) + a.RackToShards[rackKey(node.DataCenter, node.Rack)] = append(a.RackToShards[rackKey(node.DataCenter, node.Rack)], loc.ShardID) + a.NodeToShards[loc.NodeID] = append(a.NodeToShards[loc.NodeID], loc.ShardID) + a.TotalShards++ + if !contains(node.ShardIDs, loc.ShardID) { + node.ShardIDs = append(node.ShardIDs, loc.ShardID) + } +} + +// rackKey qualifies a rack name with its data center: rack names are only +// unique within a DC, so two DCs can both hold a "rack1". NUL separates the +// parts so a "/" inside either name cannot merge distinct racks. +func rackKey(dc, rack string) string { + return dc + "\x00" + rack +} + +// Finalize computes topology statistics after all nodes and shards are added. +func (a *TopologyAnalysis) Finalize() { + a.TotalNodes = len(a.AllNodes) + a.TotalRacks = len(a.RackToNodes) + a.TotalDCs = len(a.DCToRacks) +} + +func contains[T comparable](list []T, s T) bool { + for _, item := range list { + if item == s { + return true + } + } + return false +} + // ShardLocation represents where a shard is located in the topology type ShardLocation struct { ShardID int diff --git a/weed/storage/erasure_coding/distribution/config.go b/weed/storage/erasure_coding/distribution/config.go index b4935b0c71..4a0402d754 100644 --- a/weed/storage/erasure_coding/distribution/config.go +++ b/weed/storage/erasure_coding/distribution/config.go @@ -1,6 +1,14 @@ // Package distribution provides EC shard distribution algorithms with configurable EC ratios. package distribution +import ( + "fmt" + "sort" + + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" +) + // ECConfig holds erasure coding configuration parameters. // This replaces hard-coded constants like DataShardsCount=10, ParityShardsCount=4. type ECConfig struct { @@ -17,3 +25,76 @@ type ReplicationConfig struct { // Original replication string (for logging/debugging) Original string } + +// NewECConfig creates an erasure coding configuration with the given shard counts. +func NewECConfig(dataShards, parityShards int) (ECConfig, error) { + if dataShards < 1 || parityShards < 1 { + return ECConfig{}, fmt.Errorf("data and parity shard counts must be positive, got %d+%d", dataShards, parityShards) + } + return ECConfig{DataShards: dataShards, ParityShards: parityShards}, nil +} + +// DefaultECConfig returns the standard 10+4 erasure coding configuration. +func DefaultECConfig() ECConfig { + return ECConfig{DataShards: erasure_coding.DataShardsCount, ParityShards: erasure_coding.ParityShardsCount} +} + +// NewReplicationConfig parses a replication policy string like "110". +func NewReplicationConfigFromString(s string) (ReplicationConfig, error) { + rp, err := super_block.NewReplicaPlacementFromString(s) + if err != nil { + return ReplicationConfig{}, err + } + return NewReplicationConfig(rp), nil +} + +// NewReplicationConfig converts a replica placement into a replication config. +func NewReplicationConfig(rp *super_block.ReplicaPlacement) ReplicationConfig { + if rp == nil { + rp = &super_block.ReplicaPlacement{} + } + return ReplicationConfig{ + MinDataCenters: rp.DiffDataCenterCount + 1, + MinRacksPerDC: rp.DiffRackCount + 1, + MinNodesPerRack: rp.SameRackCount + 1, + Original: rp.String(), + } +} + +// TotalShards returns the number of shards each volume splits into. +func (ec ECConfig) TotalShards() int { + return ec.DataShards + ec.ParityShards +} + +// CanLoseShards returns how many shards of a volume can be missing before the +// volume is no longer readable. +func (ec ECConfig) CanLoseShards() int { + return ec.ParityShards +} + +// IsDataShard reports whether the shard id is a data shard, which serves reads. +func (ec ECConfig) IsDataShard(shardID int) bool { + return shardID >= 0 && shardID < ec.DataShards +} + +// IsParityShard reports whether the shard id is a parity shard, which is only +// used for reconstruction. +func (ec ECConfig) IsParityShard(shardID int) bool { + return shardID >= ec.DataShards && shardID < ec.TotalShards() +} + +// SortShardsDataFirst orders shard ids so data shards come before parity +// shards, for initial placement. +func (ec ECConfig) SortShardsDataFirst(shards []int) []int { + sorted := append([]int(nil), shards...) + sort.Ints(sorted) + return sorted +} + +// SortShardsParityFirst orders shard ids so parity shards come before data +// shards, for picking which shards to move away first when rebalancing. +func (ec ECConfig) SortShardsParityFirst(shards []int) []int { + sorted := append([]int(nil), shards...) + sort.Sort(sort.Reverse(sort.IntSlice(sorted))) + return sorted +} diff --git a/weed/storage/erasure_coding/distribution/distribution.go b/weed/storage/erasure_coding/distribution/distribution.go index 1ef05c55d6..3cf38c5bdc 100644 --- a/weed/storage/erasure_coding/distribution/distribution.go +++ b/weed/storage/erasure_coding/distribution/distribution.go @@ -1,5 +1,10 @@ package distribution +import ( + "fmt" + "strings" +) + // ECDistribution represents the target distribution of EC shards // based on EC configuration and replication policy. type ECDistribution struct { @@ -20,3 +25,97 @@ type ECDistribution struct { MaxShardsPerRack int MaxShardsPerNode int } + +// CalculateDistribution computes the balanced shard counts each topology level +// should hold for the given EC ratio spread over the minimum topology the +// replication policy describes. +func CalculateDistribution(ec ECConfig, rep ReplicationConfig) *ECDistribution { + total := ec.TotalShards() + rep.MinDataCenters = max(rep.MinDataCenters, 1) + rep.MinRacksPerDC = max(rep.MinRacksPerDC, 1) + rep.MinNodesPerRack = max(rep.MinNodesPerRack, 1) + dcUnits := rep.MinDataCenters + rackUnits := dcUnits * rep.MinRacksPerDC + nodeUnits := rackUnits * rep.MinNodesPerRack + return &ECDistribution{ + ECConfig: ec, + ReplicationConfig: rep, + TargetShardsPerDC: evenShare(total, dcUnits), + TargetShardsPerRack: evenShare(total, rackUnits), + TargetShardsPerNode: evenShare(total, nodeUnits), + MaxShardsPerDC: maxShare(total, dcUnits, ec.CanLoseShards()), + MaxShardsPerRack: maxShare(total, rackUnits, ec.CanLoseShards()), + MaxShardsPerNode: maxShare(total, nodeUnits, ec.CanLoseShards()), + } +} + +// evenShare is the balanced shard count per unit at one topology level. +func evenShare(total, units int) int { + return ceilDivide(total, units) +} + +// maxShare bounds how many shards a single failure domain may hold: the even +// share plus the slack that would pad the total to the next multiple of the +// unit count. When the even share is survivable on its own, the loss budget +// caps the bound so the fault tolerance report stays true even at the max. +func maxShare(total, units, lossBudget int) int { + if units < 1 { + units = 1 + } + m := ceilDivide(total, units) + units - total%units + if m > total { + m = total + } + if evenShare(total, units) <= lossBudget && m > lossBudget { + m = lossBudget + } + return m +} + +func ceilDivide(a, b int) int { + if b < 1 { + return a + } + return (a + b - 1) / b +} + +// Summary describes the distribution plan in human-readable form. +func (d *ECDistribution) Summary() string { + var sb strings.Builder + ec := d.ECConfig + rep := d.ReplicationConfig + fmt.Fprintf(&sb, "EC Configuration: %d+%d (total: %d, can lose: %d)\n", + ec.DataShards, ec.ParityShards, ec.TotalShards(), ec.CanLoseShards()) + fmt.Fprintf(&sb, "Replication: replication=%s (DCs:%d, Racks/DC:%d, Nodes/Rack:%d)\n", + rep.Original, rep.MinDataCenters, rep.MinRacksPerDC, rep.MinNodesPerRack) + sb.WriteString("Distribution Plan:\n") + fmt.Fprintf(&sb, " Data Centers: %d (target %d shards each, max %d)\n", + rep.MinDataCenters, d.TargetShardsPerDC, d.MaxShardsPerDC) + fmt.Fprintf(&sb, " Racks per DC: %d (target %d shards each, max %d)\n", + rep.MinRacksPerDC, d.TargetShardsPerRack, d.MaxShardsPerRack) + fmt.Fprintf(&sb, " Nodes per Rack: %d (target %d shards each, max %d)\n", + rep.MinNodesPerRack, d.TargetShardsPerNode, d.MaxShardsPerNode) + return sb.String() +} + +// FaultToleranceAnalysis reports whether losing one unit at each topology +// level still leaves enough shards to reconstruct the volume. +func (d *ECDistribution) FaultToleranceAnalysis() string { + var sb strings.Builder + ec := d.ECConfig + fmt.Fprintf(&sb, "Fault Tolerance Analysis for %d+%d:\n", ec.DataShards, ec.ParityShards) + writeLevel := func(name string, target int) { + remaining := ec.TotalShards() - target + status := "SURVIVABLE ✓" + if remaining < ec.DataShards { + status = "NOT SURVIVABLE ✗" + } + fmt.Fprintf(&sb, " %s Failure: %s\n", name, status) + fmt.Fprintf(&sb, " - Losing one %s loses ~%d shards\n", name, target) + fmt.Fprintf(&sb, " - Remaining: %d shards (need %d)\n", remaining, ec.DataShards) + } + writeLevel("DC", d.TargetShardsPerDC) + writeLevel("Rack", d.TargetShardsPerRack) + writeLevel("Node", d.TargetShardsPerNode) + return sb.String() +} diff --git a/weed/storage/erasure_coding/distribution/distribution_test.go b/weed/storage/erasure_coding/distribution/distribution_test.go new file mode 100644 index 0000000000..fe6962dccf --- /dev/null +++ b/weed/storage/erasure_coding/distribution/distribution_test.go @@ -0,0 +1,500 @@ +package distribution + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefaultECConfig(t *testing.T) { + ec := DefaultECConfig() + assert.Equal(t, 10, ec.DataShards) + assert.Equal(t, 4, ec.ParityShards) + assert.Equal(t, 14, ec.TotalShards()) + assert.Equal(t, 4, ec.CanLoseShards()) + assert.True(t, ec.IsDataShard(0)) + assert.True(t, ec.IsDataShard(9)) + assert.False(t, ec.IsDataShard(10)) + assert.True(t, ec.IsParityShard(13)) + assert.False(t, ec.IsParityShard(14)) + assert.Equal(t, []int{0, 1, 10, 11}, ec.SortShardsDataFirst([]int{11, 1, 10, 0})) + assert.Equal(t, []int{13, 12, 2, 1}, ec.SortShardsParityFirst([]int{2, 13, 1, 12})) +} + +func TestNewECConfig(t *testing.T) { + ec, err := NewECConfig(8, 4) + require.NoError(t, err) + assert.Equal(t, 12, ec.TotalShards()) + + for _, bad := range [][2]int{{0, 4}, {8, 0}, {-1, 4}} { + _, err := NewECConfig(bad[0], bad[1]) + assert.Error(t, err, "config %v must be rejected", bad) + } +} + +func TestNewReplicationConfigFromString(t *testing.T) { + rep, err := NewReplicationConfigFromString("110") + require.NoError(t, err) + assert.Equal(t, 2, rep.MinDataCenters) + assert.Equal(t, 2, rep.MinRacksPerDC) + assert.Equal(t, 1, rep.MinNodesPerRack) + assert.Equal(t, "110", rep.Original) + + rep, err = NewReplicationConfigFromString("000") + require.NoError(t, err) + assert.Equal(t, 1, rep.MinDataCenters) + assert.Equal(t, 1, rep.MinRacksPerDC) + assert.Equal(t, 1, rep.MinNodesPerRack) + + _, err = NewReplicationConfigFromString("abc") + assert.Error(t, err) +} + +func TestCalculateDistribution(t *testing.T) { + rep, err := NewReplicationConfigFromString("110") + require.NoError(t, err) + dist := CalculateDistribution(DefaultECConfig(), rep) + + assert.Equal(t, 7, dist.TargetShardsPerDC) + assert.Equal(t, 4, dist.TargetShardsPerRack) + assert.Equal(t, 4, dist.TargetShardsPerNode) + assert.Equal(t, 9, dist.MaxShardsPerDC) + assert.Equal(t, 4, dist.MaxShardsPerRack) + assert.Equal(t, 4, dist.MaxShardsPerNode) + + expected := "EC Configuration: 10+4 (total: 14, can lose: 4)\n" + + "Replication: replication=110 (DCs:2, Racks/DC:2, Nodes/Rack:1)\n" + + "Distribution Plan:\n" + + " Data Centers: 2 (target 7 shards each, max 9)\n" + + " Racks per DC: 2 (target 4 shards each, max 4)\n" + + " Nodes per Rack: 1 (target 4 shards each, max 4)\n" + assert.Equal(t, expected, dist.Summary()) +} + +// The reported max may not exceed the loss budget at a survivable level: +// 8+4 over "200" lets a domain drift to 7 of 12 shards, but losing that domain +// would leave 5 < 8 — capping the max at the budget keeps SURVIVABLE true. +func TestCalculateDistributionMaxWithinLossBudget(t *testing.T) { + ec, err := NewECConfig(8, 4) + require.NoError(t, err) + rep, err := NewReplicationConfigFromString("200") + require.NoError(t, err) + dist := CalculateDistribution(ec, rep) + + assert.Equal(t, 4, dist.TargetShardsPerDC) + assert.Equal(t, 4, dist.MaxShardsPerDC) + assert.Equal(t, 4, dist.MaxShardsPerRack) + assert.Equal(t, 4, dist.MaxShardsPerNode) + + // a level whose even share already exceeds the budget keeps the slack bound + dist = CalculateDistribution(DefaultECConfig(), rep) + assert.Equal(t, 6, dist.MaxShardsPerDC) // even 5 > 4: DC loss is not survivable anyway +} + +func TestCalculateDistributionSingleDomain(t *testing.T) { + rep, err := NewReplicationConfigFromString("000") + require.NoError(t, err) + dist := CalculateDistribution(DefaultECConfig(), rep) + assert.Equal(t, 14, dist.TargetShardsPerDC) + assert.Equal(t, 14, dist.MaxShardsPerDC) +} + +func TestFaultToleranceAnalysis(t *testing.T) { + ec, err := NewECConfig(8, 4) + require.NoError(t, err) + rep, err := NewReplicationConfigFromString("200") + require.NoError(t, err) + dist := CalculateDistribution(ec, rep) + + analysis := dist.FaultToleranceAnalysis() + assert.Contains(t, analysis, "Fault Tolerance Analysis for 8+4:") + assert.Contains(t, analysis, "DC Failure: SURVIVABLE") + assert.Contains(t, analysis, "- Losing one DC loses ~4 shards") + assert.Contains(t, analysis, "- Remaining: 8 shards (need 8)") +} + +func TestFaultToleranceAnalysisNotSurvivable(t *testing.T) { + rep, err := NewReplicationConfigFromString("000") + require.NoError(t, err) + dist := CalculateDistribution(DefaultECConfig(), rep) + analysis := dist.FaultToleranceAnalysis() + assert.Contains(t, analysis, "DC Failure: NOT SURVIVABLE") + assert.Contains(t, analysis, "Rack Failure: NOT SURVIVABLE") + assert.Contains(t, analysis, "Node Failure: NOT SURVIVABLE") +} + +func makeNode(id, dc, rack string, freeSlots int, shards ...int) *TopologyNode { + return &TopologyNode{NodeID: id, DataCenter: dc, Rack: rack, FreeSlots: freeSlots, ShardIDs: shards} +} + +func TestTopologyAnalysis(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 5, 0, 1)) + a.AddNode(makeNode("n2", "dc1", "r1", 5, 2)) + a.AddNode(makeNode("n3", "dc2", "r2", 5)) + a.Finalize() + + assert.Equal(t, 3, a.TotalShards) + assert.Equal(t, 3, a.TotalNodes) + assert.Equal(t, 2, a.TotalRacks) + assert.Equal(t, 2, a.TotalDCs) + assert.Equal(t, 3, a.ShardsByDC["dc1"]) + assert.Equal(t, 0, a.ShardsByDC["dc2"]) + assert.Equal(t, []string{"r1"}, a.DCToRacks["dc1"]) + assert.Equal(t, []int{0, 1, 2}, a.RackToShards[rackKey("dc1", "r1")]) +} + +// A node registered twice must still count once in rack and shard totals. +func TestTopologyAnalysisDuplicateNode(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 5, 0, 1)) + a.AddNode(makeNode("n1", "dc1", "r1", 5, 0, 1)) + a.Finalize() + + assert.Equal(t, 1, a.TotalNodes) + assert.Len(t, a.RackToNodes[rackKey("dc1", "r1")], 1) + assert.Equal(t, 2, a.ShardsByNode["n1"]) +} + +// A node auto-registered by a shard location gets its real capacity when the +// node itself registers later, plus any shards the location did not mention. +func TestTopologyAnalysisNodeCapacityLateRegistration(t *testing.T) { + a := NewTopologyAnalysis() + a.AddShardLocation(ShardLocation{ShardID: 0, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.AddNode(makeNode("n1", "dc1", "r1", 10, 1)) + a.Finalize() + + assert.Equal(t, 10, a.AllNodes["n1"].FreeSlots) + assert.Equal(t, []int{0, 1}, a.AllNodes["n1"].ShardIDs) + assert.Equal(t, 2, a.ShardsByNode["n1"]) + assert.Len(t, a.RackToNodes[rackKey("dc1", "r1")], 1) +} + +// Shards may be registered either through TopologyNode.ShardIDs or through +// AddShardLocation — both must leave the node holding the shards. +func TestTopologyAnalysisAddShardLocation(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 5)) + a.AddShardLocation(ShardLocation{ShardID: 0, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.AddShardLocation(ShardLocation{ShardID: 1, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.AddShardLocation(ShardLocation{ShardID: 1, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.Finalize() + + assert.Equal(t, 2, a.TotalShards) + assert.Equal(t, 2, a.ShardsByNode["n1"]) + assert.Equal(t, []int{0, 1}, a.AllNodes["n1"].ShardIDs) +} + +// Rack names are only unique within a DC, so dc1/rack1 and dc2/rack1 are +// distinct domains. +func TestTopologyAnalysisSameRackNameAcrossDCs(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "rack1", 5, 0, 1)) + a.AddNode(makeNode("n2", "dc2", "rack1", 5, 2)) + a.Finalize() + + assert.Equal(t, 2, a.TotalDCs) + assert.Equal(t, 2, a.TotalRacks) + assert.Equal(t, 2, a.ShardsByRack[rackKey("dc1", "rack1")]) + assert.Equal(t, 1, a.ShardsByRack[rackKey("dc2", "rack1")]) +} + +// A "/" inside a data-center or rack name must not merge distinct racks. +func TestTopologyAnalysisSlashInNames(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "a", "b/c", 5, 0)) + a.AddNode(makeNode("n2", "a/b", "c", 5, 1)) + a.Finalize() + + assert.Equal(t, 2, a.TotalRacks) + assert.Equal(t, 1, a.ShardsByRack[rackKey("a", "b/c")]) + assert.Equal(t, 1, a.ShardsByRack[rackKey("a/b", "c")]) +} + +func TestPlanRebalanceAcrossDCs(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("a1", "dcA", "rA1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("a2", "dcA", "rA2", 10, 4, 5, 6, 7)) + a.AddNode(makeNode("a3", "dcA", "rA3", 10, 8, 9, 10, 11)) + a.AddNode(makeNode("a4", "dcA", "rA4", 10, 12, 13)) + a.AddNode(makeNode("b1", "dcB", "rB1", 10)) + a.AddNode(makeNode("b2", "dcB", "rB2", 10)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("110") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + assert.Equal(t, 7, plan.MovesAcrossDC) + assert.Equal(t, 7, plan.Analysis.ShardsByDC["dcA"]) + assert.Equal(t, 7, plan.Analysis.ShardsByDC["dcB"]) + + // the given analysis still describes the un-moved layout + assert.Equal(t, 14, a.ShardsByDC["dcA"]) + assert.Equal(t, 0, a.ShardsByDC["dcB"]) + assert.Equal(t, []int{0, 1, 2, 3}, a.AllNodes["a1"].ShardIDs) + + // parity shards (10-13) move before data shards + assert.True(t, plan.Moves[0].ShardID >= 10) + assert.Equal(t, "b1", plan.Moves[0].DestNode.NodeID) +} + +func TestPlanRebalanceAcrossRacks(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("n2", "dc1", "r2", 10)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("110") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + assert.Equal(t, 2, plan.TotalMoves) + assert.Equal(t, 2, plan.MovesAcrossRack) + assert.Equal(t, 2, plan.Analysis.ShardsByRack[rackKey("dc1", "r1")]) + assert.Equal(t, 2, plan.Analysis.ShardsByRack[rackKey("dc1", "r2")]) + // parity-first ordering: shards 3 then 2 move out of r1 + assert.Equal(t, 3, plan.Moves[0].ShardID) + assert.Equal(t, 2, plan.Moves[1].ShardID) +} + +func TestPlanRebalanceWithinRack(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("n2", "dc1", "r1", 10)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("001") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + assert.Equal(t, 2, plan.TotalMoves) + assert.Equal(t, 2, plan.MovesWithinRack) + assert.Equal(t, 2, plan.Analysis.ShardsByNode["n1"]) + assert.Equal(t, 2, plan.Analysis.ShardsByNode["n2"]) + assert.Equal(t, []int{3, 2}, plan.Analysis.AllNodes["n2"].ShardIDs) + assert.Equal(t, []int{0, 1, 2, 3}, a.AllNodes["n1"].ShardIDs) +} + +func TestPlanRebalanceRespectsFreeSlots(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("n2", "dc1", "r1", 1)) + a.Finalize() + + plan, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{MinDataCenters: 1, MinRacksPerDC: 1, MinNodesPerRack: 2}).PlanRebalance(a) + require.NoError(t, err) + + assert.Equal(t, 1, plan.TotalMoves, "only one slot is free on the destination") + assert.Equal(t, 3, plan.Analysis.ShardsByNode["n1"]) + assert.Equal(t, 1, plan.Analysis.ShardsByNode["n2"]) + assert.Equal(t, 0, plan.Analysis.AllNodes["n2"].FreeSlots) + assert.Equal(t, 1, a.AllNodes["n2"].FreeSlots) +} + +func TestPlanRebalanceNoopWhenBalanced(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1)) + a.AddNode(makeNode("n2", "dc1", "r1", 10, 2, 3)) + a.Finalize() + + plan, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{MinDataCenters: 1, MinRacksPerDC: 1, MinNodesPerRack: 2}).PlanRebalance(a) + require.NoError(t, err) + assert.Equal(t, 0, plan.TotalMoves) +} + +func TestPlanRebalanceNilAnalysis(t *testing.T) { + _, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{}).PlanRebalance(nil) + assert.Error(t, err) +} + +// Node balancing inside a rack must aim for the rack's count after +// DC- and rack-level moves, not the count the rack started with. +func TestPlanRebalanceNodeTargetAfterMoves(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)) + a.AddNode(makeNode("n2", "dc1", "r1", 10)) + a.AddNode(makeNode("n3", "dc1", "r2", 10)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("010") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + assert.Equal(t, 7, plan.Analysis.ShardsByRack[rackKey("dc1", "r1")]) + assert.Equal(t, 7, plan.Analysis.ShardsByRack[rackKey("dc1", "r2")]) + assert.Equal(t, 4, plan.Analysis.ShardsByNode["n1"]) + assert.Equal(t, 3, plan.Analysis.ShardsByNode["n2"]) + assert.Equal(t, []int{0, 1, 2, 3}, plan.Analysis.AllNodes["n1"].ShardIDs) + assert.Equal(t, []int{6, 5, 4}, plan.Analysis.AllNodes["n2"].ShardIDs) +} + +// A shard that already has a copy anywhere in the destination domain must not +// be moved there again — the second copy adds no redundancy. +func TestPlanRebalanceNoDuplicateShardInDomain(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("a1", "dcA", "rA1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("b1", "dcB", "rB1", 10, 3)) + a.AddNode(makeNode("b2", "dcB", "rB2", 10)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("100") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + // dcA holds 4, dcB holds 1, target 3 each: one move, and it cannot be shard 3 + require.Len(t, plan.Moves, 1) + assert.Equal(t, 2, plan.Moves[0].ShardID) + assert.Equal(t, []int{2}, plan.Analysis.AllNodes["b2"].ShardIDs) + assert.Equal(t, []int{3}, plan.Analysis.AllNodes["b1"].ShardIDs) +} + +// Shard locations may be registered without registering the node first; the +// rebalancer must still see the shard, counted under the node's own dc/rack. +func TestPlanRebalanceShardForUnregisteredNode(t *testing.T) { + a := NewTopologyAnalysis() + a.AddShardLocation(ShardLocation{ShardID: 0, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.AddShardLocation(ShardLocation{ShardID: 1, NodeID: "n1", DataCenter: "dc1", Rack: "r1"}) + a.AddNode(makeNode("n2", "dc1", "r1", 10)) + a.Finalize() + + assert.Equal(t, 1, a.TotalDCs) + assert.Equal(t, 2, a.ShardsByNode["n1"]) + assert.Equal(t, []int{0, 1}, a.AllNodes["n1"].ShardIDs) + + plan, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{MinDataCenters: 1, MinRacksPerDC: 1, MinNodesPerRack: 2}).PlanRebalance(a) + require.NoError(t, err) + assert.Equal(t, 1, plan.TotalMoves) + assert.Equal(t, 1, plan.Analysis.ShardsByNode["n1"]) + assert.Equal(t, 1, plan.Analysis.ShardsByNode["n2"]) +} + +// When the most overloaded domain can only offer duplicates, balancing must +// fall through to the next overloaded source rather than stop. +func TestPlanRebalanceSkipsBlockedSource(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("a1", "dcA", "rA", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("a2", "dcA", "rA", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("a3", "dcA", "rA", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("b1", "dcB", "rB", 10, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)) + a.AddNode(makeNode("c1", "dcC", "rC", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("c2", "dcC", "rC", 10)) + a.AddNode(makeNode("d1", "dcD", "rD", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("d2", "dcD", "rD", 10)) + a.Finalize() + + plan, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{MinDataCenters: 4, MinRacksPerDC: 1, MinNodesPerRack: 1}).PlanRebalance(a) + require.NoError(t, err) + + // dcA is stuck on duplicates (target 8), but dcB still sheds two shards + assert.Equal(t, 12, plan.Analysis.ShardsByDC["dcA"]) + assert.Equal(t, 8, plan.Analysis.ShardsByDC["dcB"]) + assert.Equal(t, 5, plan.Analysis.ShardsByDC["dcC"]) + assert.Equal(t, 5, plan.Analysis.ShardsByDC["dcD"]) +} + +// A shard that dc-level balancing lands on a node can be shed again at rack or +// node level; its moves must collapse to one transfer to the final holder. +func TestPlanRebalanceCollapsesChainedMoves(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("a1", "dcA", "rA", 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)) + a.AddNode(makeNode("b1", "dcB", "rB1", 20)) + a.AddNode(makeNode("c1", "dcB", "rB2", 20)) + a.AddNode(makeNode("c2", "dcB", "rB2", 20)) + a.AddNode(makeNode("c3", "dcB", "rB2", 20)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("100") + require.NoError(t, err) + plan, err := NewRebalancer(DefaultECConfig(), rep).PlanRebalance(a) + require.NoError(t, err) + + moved := make(map[int]string) + for _, m := range plan.Moves { + _, twice := moved[m.ShardID] + assert.False(t, twice, "shard %d moves more than once", m.ShardID) + moved[m.ShardID] = m.DestNode.NodeID + assert.Contains(t, m.DestNode.ShardIDs, m.ShardID) + } + assert.Equal(t, 7, plan.TotalMoves) + assert.Equal(t, 7, plan.MovesAcrossDC) + // shard 12 reached b1 through c1 in simulation; the plan moves it directly + assert.Equal(t, "b1", moved[12]) + assert.Equal(t, []int{13, 9, 12}, plan.Analysis.AllNodes["b1"].ShardIDs) +} + +// A collapsed move must not be scheduled before the departures that free its +// destination: applying the plan in order may never target a full node. +func TestPlanRebalanceMovesRespectSlots(t *testing.T) { + ec, err := NewECConfig(20, 4) + require.NoError(t, err) + a := NewTopologyAnalysis() + a.AddNode(makeNode("a1", "dcA", "rA", 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)) + a.AddNode(makeNode("b1", "dcB", "rB1", 10)) + a.AddNode(makeNode("b2", "dcB", "rB1", 0, 20, 21, 22, 23)) + a.AddNode(makeNode("c1", "dcB", "rB2", 10, 17, 18, 19)) + a.Finalize() + + rep, err := NewReplicationConfigFromString("100") + require.NoError(t, err) + plan, err := NewRebalancer(ec, rep).PlanRebalance(a) + require.NoError(t, err) + + free := make(map[string]int) + for _, n := range a.AllNodes { + free[n.NodeID] = n.FreeSlots + } + firstDeparture, firstArrival := -1, -1 + for i, m := range plan.Moves { + assert.Greater(t, free[m.DestNode.NodeID], 0, "move %d targets a full node", i) + free[m.DestNode.NodeID]-- + free[m.SourceNode.NodeID]++ + if m.DestNode.NodeID == "b2" && firstArrival < 0 { + firstArrival = i + } + if m.SourceNode.NodeID == "b2" && firstDeparture < 0 { + firstDeparture = i + } + } + moved := make(map[int]string) + for _, m := range plan.Moves { + moved[m.ShardID] = m.DestNode.NodeID + } + // shard 16 reaches full-at-start b2 through b1; the plan moves it directly, + // but only after the departures that free b2's slots + assert.Equal(t, "b2", moved[16]) + assert.Equal(t, "c1", moved[23]) + assert.Greater(t, firstArrival, firstDeparture) +} + +func TestPlanRebalanceNoDuplicateShardOnNode(t *testing.T) { + a := NewTopologyAnalysis() + a.AddNode(makeNode("n1", "dc1", "r1", 10, 0, 1, 2, 3)) + a.AddNode(makeNode("n2", "dc1", "r1", 10, 3)) + a.Finalize() + + plan, err := NewRebalancer(DefaultECConfig(), ReplicationConfig{MinDataCenters: 1, MinRacksPerDC: 1, MinNodesPerRack: 2}).PlanRebalance(a) + require.NoError(t, err) + + // n2 already holds 3, so the balancing move brings a different shard + assert.Equal(t, 3, plan.Analysis.ShardsByNode["n1"]) + assert.Equal(t, 2, plan.Analysis.ShardsByNode["n2"]) + assert.Equal(t, []int{3, 2}, plan.Analysis.AllNodes["n2"].ShardIDs) + for _, move := range plan.Moves { + assert.NotContains(t, move.DestNode.ShardIDs[:len(move.DestNode.ShardIDs)-1], move.ShardID, + "shard %d must not be moved onto a node already holding it", move.ShardID) + } +} + +func TestCalculateDistributionZeroReplication(t *testing.T) { + dist := CalculateDistribution(DefaultECConfig(), ReplicationConfig{}) + assert.Equal(t, 14, dist.TargetShardsPerDC) + assert.Contains(t, dist.Summary(), "Data Centers: 1") + assert.NotPanics(t, func() { dist.FaultToleranceAnalysis() }) +} diff --git a/weed/storage/erasure_coding/distribution/rebalancer.go b/weed/storage/erasure_coding/distribution/rebalancer.go index 2442e59a97..7ee178783b 100644 --- a/weed/storage/erasure_coding/distribution/rebalancer.go +++ b/weed/storage/erasure_coding/distribution/rebalancer.go @@ -1,5 +1,10 @@ package distribution +import ( + "fmt" + "sort" +) + // ShardMove represents a planned shard move type ShardMove struct { ShardID int @@ -12,7 +17,7 @@ type ShardMove struct { type RebalancePlan struct { Moves []ShardMove Distribution *ECDistribution - Analysis *TopologyAnalysis + Analysis *TopologyAnalysis // simulated layout once every move applies; move endpoints reference its nodes // Statistics TotalMoves int @@ -26,3 +31,305 @@ type Rebalancer struct { ecConfig ECConfig repConfig ReplicationConfig } + +// NewRebalancer creates a rebalancer for the given EC ratio and replication policy. +func NewRebalancer(ec ECConfig, rep ReplicationConfig) *Rebalancer { + return &Rebalancer{ecConfig: ec, repConfig: rep} +} + +// PlanRebalance generates shard moves that bring the volume's shard layout to +// the balanced distribution: first across data centers, then racks within each +// data center, then nodes within each rack. Moves stop when source and +// destination both reach their even share, or when destination nodes have no +// free slots. The moves are planned on a copy of the topology, so the given +// analysis still describes the cluster as it is; plan.Analysis shows the +// layout the moves would produce. +func (r *Rebalancer) PlanRebalance(analysis *TopologyAnalysis) (*RebalancePlan, error) { + if analysis == nil { + return nil, fmt.Errorf("cannot plan rebalancing without a topology analysis") + } + sim := NewTopologyAnalysis() + for _, id := range sortedKeys(analysis.AllNodes) { + node := *analysis.AllNodes[id] + node.ShardIDs = append([]int(nil), node.ShardIDs...) + sim.AddNode(&node) + } + sim.Finalize() + plan := &RebalancePlan{ + Distribution: CalculateDistribution(r.ecConfig, r.repConfig), + Analysis: sim, + } + + recordMove := func(shardID int, src, dst *TopologyNode, level, domain string) { + src.ShardIDs = removeShard(src.ShardIDs, shardID) + dst.ShardIDs = append(dst.ShardIDs, shardID) + src.FreeSlots++ + dst.FreeSlots-- + sim.ShardsByDC[src.DataCenter]-- + sim.ShardsByDC[dst.DataCenter]++ + sim.ShardsByRack[rackKey(src.DataCenter, src.Rack)]-- + sim.ShardsByRack[rackKey(dst.DataCenter, dst.Rack)]++ + sim.ShardsByNode[src.NodeID]-- + sim.ShardsByNode[dst.NodeID]++ + sim.DCToShards[src.DataCenter] = removeShard(sim.DCToShards[src.DataCenter], shardID) + sim.DCToShards[dst.DataCenter] = append(sim.DCToShards[dst.DataCenter], shardID) + sim.RackToShards[rackKey(src.DataCenter, src.Rack)] = removeShard(sim.RackToShards[rackKey(src.DataCenter, src.Rack)], shardID) + sim.RackToShards[rackKey(dst.DataCenter, dst.Rack)] = append(sim.RackToShards[rackKey(dst.DataCenter, dst.Rack)], shardID) + sim.NodeToShards[src.NodeID] = removeShard(sim.NodeToShards[src.NodeID], shardID) + sim.NodeToShards[dst.NodeID] = append(sim.NodeToShards[dst.NodeID], shardID) + plan.Moves = append(plan.Moves, ShardMove{ + ShardID: shardID, + SourceNode: src, + DestNode: dst, + Reason: fmt.Sprintf("%s balance: %s above target", level, domain), + }) + } + + moveOut := func(level, srcDomain string, srcNodes, dstNodes []*TopologyNode) bool { + var shardIDs []int + for _, n := range srcNodes { + shardIDs = append(shardIDs, n.ShardIDs...) + } + for _, shardID := range r.ecConfig.SortShardsParityFirst(shardIDs) { + var src *TopologyNode + for _, n := range srcNodes { + if contains(n.ShardIDs, shardID) { + src = n + break + } + } + if src == nil { + continue + } + // a shard already inside the destination domain gains nothing there + duplicate := false + for _, n := range dstNodes { + if contains(n.ShardIDs, shardID) { + duplicate = true + break + } + } + if duplicate { + continue + } + var dst *TopologyNode + for _, n := range dstNodes { + if n.FreeSlots > 0 && (dst == nil || nodeLess(n, dst)) { + dst = n + } + } + if dst == nil { + continue + } + recordMove(shardID, src, dst, level, srcDomain) + return true + } + return false + } + + // balance moves shards between domains of one level until every domain + // holds at most its even share or no destination can take a shard. A + // source that cannot shed any shard is set aside rather than stopping + // the level, so other overloaded sources still get to move. + balance := func(level string, domains []string, countOf func(string) int, target int, nodesOf func(string) []*TopologyNode, dstDomains func(string) []string) { + blocked := make(map[string]bool) + for { + src := "" + srcFound := false + for _, d := range domains { + if blocked[d] { + continue + } + if countOf(d) > target && (!srcFound || countOf(d) > countOf(src)) { + src, srcFound = d, true + } + } + if !srcFound { + return + } + dsts := append([]string(nil), dstDomains(src)...) + sort.Slice(dsts, func(i, j int) bool { + if countOf(dsts[i]) != countOf(dsts[j]) { + return countOf(dsts[i]) < countOf(dsts[j]) + } + return dsts[i] < dsts[j] + }) + moved := false + for _, dst := range dsts { + if countOf(dst) < target && moveOut(level, src, nodesOf(src), nodesOf(dst)) { + moved = true + break + } + } + if !moved { + blocked[src] = true + } + } + } + + dcList := sortedKeys(sim.DCToRacks) + dcRackKeys := func(dc string) []string { + racks := sim.DCToRacks[dc] + keys := make([]string, 0, len(racks)) + for _, rack := range racks { + keys = append(keys, rackKey(dc, rack)) + } + return keys + } + totalShards := 0 + for _, dc := range dcList { + totalShards += shardCount(dcRackKeys(dc), sim.RackToNodes) + } + dcCount := func(dc string) int { return shardCount(dcRackKeys(dc), sim.RackToNodes) } + balance("dc", dcList, dcCount, ceilDivide(totalShards, len(dcList)), + func(dc string) []*TopologyNode { return nodesOfRacks(dcRackKeys(dc), sim.RackToNodes) }, + func(string) []string { return dcList }) + + for _, dc := range dcList { + racks := sim.DCToRacks[dc] + rackCount := func(rack string) int { return shardCount([]string{rackKey(dc, rack)}, sim.RackToNodes) } + balance("rack", racks, rackCount, ceilDivide(dcCount(dc), len(racks)), + func(rack string) []*TopologyNode { return sim.RackToNodes[rackKey(dc, rack)] }, + func(string) []string { return racks }) + } + + for _, rack := range sortedKeys(sim.RackToNodes) { + nodes := sim.RackToNodes[rack] + nodeCount := func(nodeID string) int { + for _, n := range nodes { + if n.NodeID == nodeID { + return len(n.ShardIDs) + } + } + return 0 + } + nodeIDs := make([]string, 0, len(nodes)) + for _, n := range nodes { + nodeIDs = append(nodeIDs, n.NodeID) + } + sort.Strings(nodeIDs) + balance("node", nodeIDs, nodeCount, ceilDivide(shardCount([]string{rack}, sim.RackToNodes), len(nodes)), + func(nodeID string) []*TopologyNode { + for _, n := range nodes { + if n.NodeID == nodeID { + return []*TopologyNode{n} + } + } + return nil + }, + func(string) []string { return nodeIDs }) + } + + // A copy of a shard moves at most once per level, and each finer level only + // moves it deeper inside the domain the previous level sent it to, so its + // moves form a chain from the first source to the final destination. + // Collapse each chain into one transfer — intermediate hops would copy the + // shard over the network only to move it again. Chains are keyed by where + // the copy sits, so duplicate copies of a shard still move independently. + pending := make(map[ShardLocation]int) + moves := plan.Moves[:0] + for _, m := range plan.Moves { + key := ShardLocation{ShardID: m.ShardID, NodeID: m.SourceNode.NodeID} + if i, ok := pending[key]; ok { + delete(pending, key) + moves[i].DestNode = m.DestNode + pending[ShardLocation{ShardID: m.ShardID, NodeID: m.DestNode.NodeID}] = i + continue + } + pending[ShardLocation{ShardID: m.ShardID, NodeID: m.DestNode.NodeID}] = len(moves) + moves = append(moves, m) + } + // Order the collapsed moves so each destination has a free slot when its + // move is applied: a chain's final destination may only gain a slot from a + // later move's departure, so the early position the collapse kept can be + // wrong. Emit every move whose destination has room first; a source gains + // its slot back as soon as its move is emitted. + free := make(map[string]int) + for _, n := range sim.AllNodes { + free[n.NodeID] = n.FreeSlots + } + for _, m := range moves { + free[m.DestNode.NodeID]++ + free[m.SourceNode.NodeID]-- + } + ordered := make([]ShardMove, 0, len(moves)) + for len(moves) > 0 { + kept := moves[:0] + for _, m := range moves { + if free[m.DestNode.NodeID] > 0 { + free[m.DestNode.NodeID]-- + free[m.SourceNode.NodeID]++ + ordered = append(ordered, m) + } else { + kept = append(kept, m) + } + } + if len(kept) == len(moves) { + ordered = append(ordered, kept...) + break + } + moves = kept + } + plan.Moves = ordered + for _, m := range ordered { + switch { + case m.SourceNode.DataCenter != m.DestNode.DataCenter: + plan.MovesAcrossDC++ + case m.SourceNode.Rack != m.DestNode.Rack: + plan.MovesAcrossRack++ + default: + plan.MovesWithinRack++ + } + } + plan.TotalMoves = len(ordered) + return plan, nil +} + +// nodeLess prefers the node holding fewer of the volume's shards, then the one +// with more free slots, for a stable destination choice. +func nodeLess(a, b *TopologyNode) bool { + if len(a.ShardIDs) != len(b.ShardIDs) { + return len(a.ShardIDs) < len(b.ShardIDs) + } + if a.FreeSlots != b.FreeSlots { + return a.FreeSlots > b.FreeSlots + } + return a.NodeID < b.NodeID +} + +// shardCount sums the shards the given racks currently hold for the volume. +func shardCount(racks []string, rackToNodes map[string][]*TopologyNode) int { + total := 0 + for _, rack := range racks { + for _, n := range rackToNodes[rack] { + total += len(n.ShardIDs) + } + } + return total +} + +func nodesOfRacks(racks []string, rackToNodes map[string][]*TopologyNode) []*TopologyNode { + var nodes []*TopologyNode + for _, rack := range racks { + nodes = append(nodes, rackToNodes[rack]...) + } + return nodes +} + +func removeShard(shards []int, shardID int) []int { + for i, s := range shards { + if s == shardID { + return append(shards[:i], shards[i+1:]...) + } + } + return shards +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/weed/storage/store.go b/weed/storage/store.go index 0186b0d269..0d2b51b50d 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -21,6 +21,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" @@ -1292,6 +1293,11 @@ func (s *Store) deleteVolumeGuarded(i needle.VolumeId, onlyEmpty bool, onlyGarba func (s *Store) ConfigureVolume(i needle.VolumeId, replication string) error { + rp, err := super_block.NewReplicaPlacementFromString(replication) + if err != nil { + return fmt.Errorf("volume %d replication %q: %v", i, replication, err) + } + for _, location := range s.Locations { fileInfo, found := location.LocateVolume(i) if !found { @@ -1309,12 +1315,42 @@ func (s *Store) ConfigureVolume(i needle.VolumeId, replication string) error { if err != nil { return fmt.Errorf("volume %d failed to save vif: %v", i, err) } - return nil + if v := s.GetVolume(i); v != nil { + return v.UpdateReplicaPlacement(rp) + } + return updateDatReplicaPlacement(filepath.Join(location.Directory, baseFileName+".dat"), i, rp) } return fmt.Errorf("volume %d not found on disk", i) } +// updateDatReplicaPlacement rewrites the replica placement in a .dat header +// for a volume that is not mounted; a mounted volume goes through +// UpdateReplicaPlacement so the change lands under its data-file locks. +func updateDatReplicaPlacement(datFileName string, vid needle.VolumeId, rp *super_block.ReplicaPlacement) error { + f, err := os.OpenFile(datFileName, os.O_RDWR, 0644) + if err != nil { + if os.IsNotExist(err) { + return nil // remote-only volume: the .vif carries the placement + } + return fmt.Errorf("volume %d failed to open %s: %v", vid, datFileName, err) + } + defer f.Close() + datBackend := backend.NewDiskFile(f) + superBlock, err := super_block.ReadSuperBlock(datBackend) + if err != nil { + return fmt.Errorf("volume %d failed to parse superblock: %v", vid, err) + } + superBlock.ReplicaPlacement = rp + if _, err := datBackend.WriteAt(superBlock.Bytes(), 0); err != nil { + return fmt.Errorf("volume %d failed to write superblock: %v", vid, err) + } + if err := datBackend.Sync(); err != nil { + return fmt.Errorf("volume %d failed to sync superblock: %v", vid, err) + } + return nil +} + func (s *Store) SetVolumeSizeLimit(x uint64) { s.volumeSizeLimit.Store(x) } diff --git a/weed/storage/store_configure_volume_test.go b/weed/storage/store_configure_volume_test.go new file mode 100644 index 0000000000..8cb4c85629 --- /dev/null +++ b/weed/storage/store_configure_volume_test.go @@ -0,0 +1,47 @@ +package storage + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// volume.configure must leave the live superblock, the .dat header, the +// in-memory .vif copy and the .vif on disk all on the new placement. +func TestConfigureVolumeUpdatesSuperblock(t *testing.T) { + dir := t.TempDir() + store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{dir}, []int32{10}, []util.MinFreeSpace{{}}, "", NeedleMapInMemory, + []types.DiskType{types.HardDriveType}, nil, 0, stats.DefaultDiskIOProbeConfig()) + t.Cleanup(store.Close) + require.NoError(t, store.AddVolume(1, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0)) + + require.NoError(t, store.ConfigureVolume(1, "010")) + + v := store.GetVolume(1) + require.NotNil(t, v) + require.Equal(t, "010", v.ReplicaPlacement.String()) + require.Equal(t, "010", v.GetVolumeInfo().GetReplication()) + + datFile, err := os.Open(v.FileName(".dat")) + require.NoError(t, err) + sb, err := super_block.ReadSuperBlock(backend.NewDiskFile(datFile)) + require.NoError(t, err) + datFile.Close() + require.Equal(t, "010", sb.ReplicaPlacement.String()) + + volumeInfo, _, _, err := volume_info.MaybeLoadVolumeInfo(v.FileName(".vif")) + require.NoError(t, err) + require.Equal(t, "010", volumeInfo.GetReplication()) + + require.Error(t, store.ConfigureVolume(1, "bogus")) +} diff --git a/weed/storage/super_block/super_block_read.go.go b/weed/storage/super_block/super_block_read.go.go index dba1cba728..3ded9e8398 100644 --- a/weed/storage/super_block/super_block_read.go.go +++ b/weed/storage/super_block/super_block_read.go.go @@ -2,6 +2,7 @@ package super_block import ( "fmt" + "io" "google.golang.org/protobuf/proto" @@ -15,11 +16,12 @@ import ( func ReadSuperBlock(datBackend backend.BackendStorageFile) (superBlock SuperBlock, err error) { header := make([]byte, SuperBlockSize) - if n, e := datBackend.ReadAt(header, 0); e != nil { - if n != SuperBlockSize { - err = fmt.Errorf("cannot read volume %s super block: %v", datBackend.Name(), e) - return + if n, e := datBackend.ReadAt(header, 0); n != SuperBlockSize { + if e == nil { + e = io.ErrUnexpectedEOF } + err = fmt.Errorf("cannot read volume %s super block: %v", datBackend.Name(), e) + return } superBlock.Version = needle.Version(header[0]) @@ -34,10 +36,17 @@ func ReadSuperBlock(datBackend backend.BackendStorageFile) (superBlock SuperBloc if superBlock.ExtraSize > 0 { // read more extraData := make([]byte, int(superBlock.ExtraSize)) + if n, e := datBackend.ReadAt(extraData, SuperBlockSize); n != len(extraData) { + if e == nil { + e = io.ErrUnexpectedEOF + } + err = fmt.Errorf("cannot read volume %s super block extra: %v", datBackend.Name(), e) + return + } superBlock.Extra = &master_pb.SuperBlockExtra{} err = proto.Unmarshal(extraData, superBlock.Extra) if err != nil { - err = fmt.Errorf("cannot read volume %s super block extra: %v", datBackend.Name(), err) + err = fmt.Errorf("cannot unmarshal volume %s super block extra: %v", datBackend.Name(), err) return } } diff --git a/weed/storage/super_block/super_block_test.go b/weed/storage/super_block/super_block_test.go index b952240ae9..6dd1d268c2 100644 --- a/weed/storage/super_block/super_block_test.go +++ b/weed/storage/super_block/super_block_test.go @@ -1,8 +1,16 @@ package super_block import ( + "io" + "os" + "path/filepath" + "strings" "testing" + "google.golang.org/protobuf/proto" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) @@ -23,3 +31,147 @@ func TestSuperBlockReadWrite(t *testing.T) { } } + +func TestReadSuperBlockWithExtra(t *testing.T) { + rp, _ := NewReplicaPlacementFromByte(byte(001)) + ttl, _ := needle.ReadTTL("15d") + s := &SuperBlock{ + Version: needle.GetCurrentVersion(), + ReplicaPlacement: rp, + Ttl: ttl, + Extra: &master_pb.SuperBlockExtra{ + ErasureCoding: &master_pb.SuperBlockExtra_ErasureCoding{ + Data: 10, + Parity: 4, + VolumeIds: []uint32{1, 2, 3}, + }, + }, + } + + datPath := filepath.Join(t.TempDir(), "1.dat") + if err := os.WriteFile(datPath, s.Bytes(), 0644); err != nil { + t.Fatal(err) + } + f, err := os.Open(datPath) + if err != nil { + t.Fatal(err) + } + datBackend := backend.NewDiskFile(f) + defer datBackend.Close() + + got, err := ReadSuperBlock(datBackend) + if err != nil { + t.Fatalf("ReadSuperBlock: %v", err) + } + if got.ExtraSize != s.ExtraSize { + t.Errorf("ExtraSize = %d, want %d", got.ExtraSize, s.ExtraSize) + } + if !proto.Equal(got.Extra, s.Extra) { + t.Errorf("Extra = %v, want %v", got.Extra, s.Extra) + } +} + +// io.ReaderAt may return io.EOF on a full read; a remote backend can pass that through. +type eofBackendStorageFile struct { + backend.DiskFile + data []byte +} + +func (f *eofBackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) { + if off >= int64(len(f.data)) { + return 0, io.EOF + } + return copy(p, f.data[off:]), io.EOF +} + +func TestReadSuperBlockExtraFullReadWithEOF(t *testing.T) { + rp, _ := NewReplicaPlacementFromByte(byte(001)) + ttl, _ := needle.ReadTTL("15d") + s := &SuperBlock{ + Version: needle.GetCurrentVersion(), + ReplicaPlacement: rp, + Ttl: ttl, + Extra: &master_pb.SuperBlockExtra{ + ErasureCoding: &master_pb.SuperBlockExtra_ErasureCoding{ + Data: 10, + Parity: 4, + VolumeIds: []uint32{1, 2, 3}, + }, + }, + } + + got, err := ReadSuperBlock(&eofBackendStorageFile{data: s.Bytes()}) + if err != nil { + t.Fatalf("ReadSuperBlock: %v", err) + } + if !proto.Equal(got.Extra, s.Extra) { + t.Errorf("Extra = %v, want %v", got.Extra, s.Extra) + } +} + +func TestReadSuperBlockTruncatedExtra(t *testing.T) { + rp, _ := NewReplicaPlacementFromByte(byte(001)) + ttl, _ := needle.ReadTTL("15d") + s := &SuperBlock{ + Version: needle.GetCurrentVersion(), + ReplicaPlacement: rp, + Ttl: ttl, + Extra: &master_pb.SuperBlockExtra{ + ErasureCoding: &master_pb.SuperBlockExtra_ErasureCoding{ + Data: 10, + Parity: 4, + VolumeIds: []uint32{1, 2, 3}, + }, + }, + } + + data := s.Bytes() + datPath := filepath.Join(t.TempDir(), "1.dat") + if err := os.WriteFile(datPath, data[:len(data)-1], 0644); err != nil { + t.Fatal(err) + } + f, err := os.Open(datPath) + if err != nil { + t.Fatal(err) + } + datBackend := backend.NewDiskFile(f) + defer datBackend.Close() + + _, err = ReadSuperBlock(datBackend) + if err == nil || !strings.Contains(err.Error(), "cannot read") { + t.Fatalf("expected read error for truncated extra, got %v", err) + } +} + +// s3_backend.ReadAt clears io.EOF even when it returns fewer bytes than requested. +type shortReadBackendStorageFile struct { + backend.DiskFile + data []byte +} + +func (f *shortReadBackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) { + return copy(p, f.data[off:]), nil +} + +func TestReadSuperBlockShortReadWithoutError(t *testing.T) { + rp, _ := NewReplicaPlacementFromByte(byte(001)) + ttl, _ := needle.ReadTTL("15d") + s := &SuperBlock{ + Version: needle.GetCurrentVersion(), + ReplicaPlacement: rp, + Ttl: ttl, + Extra: &master_pb.SuperBlockExtra{ + ErasureCoding: &master_pb.SuperBlockExtra_ErasureCoding{ + Data: 10, + Parity: 4, + VolumeIds: []uint32{1, 2, 3}, + }, + }, + } + + data := s.Bytes() + _, err := ReadSuperBlock(&shortReadBackendStorageFile{data: data[:len(data)-1]}) + if err == nil || !strings.Contains(err.Error(), "cannot read") { + t.Fatalf("expected read error for short extra read, got %v", err) + } +} diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 553435e5b8..d92a930309 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -188,6 +188,35 @@ func (v *Volume) Version() needle.Version { return v.SuperBlock.Version } +// UpdateReplicaPlacement records rp in the live volume — the in-memory .vif +// copy and the superblock — and rewrites the .dat header when the data file +// is local. Keeps a mounted volume consistent with the .vif the caller just +// configured on disk, so heartbeats and later compactions report and rewrite +// the same placement. +func (v *Volume) UpdateReplicaPlacement(rp *super_block.ReplicaPlacement) error { + v.dataFileSyncLock.Lock() + defer v.dataFileSyncLock.Unlock() + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() + v.superBlockAccessLock.Lock() + defer v.superBlockAccessLock.Unlock() + v.volumeInfoRWLock.Lock() + defer v.volumeInfoRWLock.Unlock() + + v.SuperBlock.ReplicaPlacement = rp + if v.volumeInfo != nil { + v.volumeInfo.Replication = rp.String() + } + diskFile, ok := v.DataBackend.(*backend.DiskFile) + if !ok { + return nil // remote-tiered: the .vif carries the placement + } + if _, err := diskFile.WriteAt(v.SuperBlock.Bytes(), 0); err != nil { + return fmt.Errorf("volume %d failed to write superblock: %v", v.Id, err) + } + return diskFile.Sync() +} + func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) { v.dataFileAccessLock.RLock() defer v.dataFileAccessLock.RUnlock() diff --git a/weed/storage/volume_tier.go b/weed/storage/volume_tier.go index 7baa94b766..cd046fdd01 100644 --- a/weed/storage/volume_tier.go +++ b/weed/storage/volume_tier.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -142,6 +143,41 @@ func (v *Volume) loadRemoteFileLocked() error { return nil } +// AttachRemoteDatFile records remoteFile, a copy of the .dat read from source, +// in the volume's .vif and switches the data backend to it. It refuses when +// source is no longer the volume's data backend: a compaction commit since the +// copy replaced the .dat and the index, and the copy has the old layout. The +// check, the .vif write and the switch happen under the data-file locks, so a +// commit either lands before (and is seen here) or after (and refuses a tiered +// volume). +func (v *Volume) AttachRemoteDatFile(source backend.BackendStorageFile, remoteFile *volume_server_pb.RemoteFile) error { + v.dataFileSyncLock.Lock() + defer v.dataFileSyncLock.Unlock() + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() + if v.DataBackend != source { + return fmt.Errorf("volume %d data file changed during the copy (compacted or reloaded)", v.Id) + } + files := v.volumeInfo.GetFiles() + v.volumeInfo.Files = append(files, remoteFile) + if err := v.SaveVolumeInfo(); err != nil { + // A NotCrashDurableError means the rename committed: the .vif names + // the copy, so keep the entry and serve from the copy, and still + // report the error so the caller keeps the local .dat. Any other + // failure left the old .vif in place. + var ndErr *volume_info.NotCrashDurableError + if !errors.As(err, &ndErr) { + v.volumeInfo.Files = files + return fmt.Errorf("save remote file info: %w", err) + } + if loadErr := v.loadRemoteFileLocked(); loadErr != nil { + return fmt.Errorf("save remote file info: %w; load remote file: %v", err, loadErr) + } + return fmt.Errorf("save remote file info: %w", err) + } + return v.loadRemoteFileLocked() +} + func (v *Volume) SaveVolumeInfo() error { tierFileName := v.FileName(".vif") @@ -149,6 +185,9 @@ func (v *Volume) SaveVolumeInfo() error { v.volumeInfo.ExpireAtSec = expireAtSec } - return volume_info.SaveVolumeInfo(tierFileName, v.volumeInfo) + return saveVolumeInfoFile(tierFileName, v.volumeInfo) } + +// saveVolumeInfoFile writes a .vif; tests replace it to inject failures. +var saveVolumeInfoFile = volume_info.SaveVolumeInfo diff --git a/weed/storage/volume_tier_attach_test.go b/weed/storage/volume_tier_attach_test.go new file mode 100644 index 0000000000..0322c591bb --- /dev/null +++ b/weed/storage/volume_tier_attach_test.go @@ -0,0 +1,70 @@ +package storage + +import ( + "errors" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" +) + +// When the .vif rename commits but the directory fsync fails, the saved .vif +// names the remote copy. AttachRemoteDatFile must keep that entry (the upload +// handler deletes a copy the volume info does not name) and serve from the +// copy the .vif points at, while still reporting the error. +func TestAttachRemoteDatFileKeepsACommittedVifEntry(t *testing.T) { + b := newLocalDirBackend(t) + registerTestBackend(t, b) + + dir := t.TempDir() + v, err := NewVolume(dir, dir, "", 86, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + require.NoError(t, err) + defer v.Close() + for i := 1; i <= 5; i++ { + _, _, _, err := v.writeNeedle2(newRandomNeedle(uint64(i)), true, false, false) + require.NoError(t, err) + } + diskFile, ok := v.DataBackend.(*backend.DiskFile) + require.True(t, ok) + key, size, err := b.CopyFile(diskFile.File, nil, 0) + require.NoError(t, err) + + saved := saveVolumeInfoFile + saveVolumeInfoFile = func(fileName string, info *volume_server_pb.VolumeInfo) error { + if err := saved(fileName, info); err != nil { + return err + } + return &volume_info.NotCrashDurableError{FileName: fileName, Err: syscall.EIO} + } + defer func() { saveVolumeInfoFile = saved }() + + bType, bId := backend.BackendNameToTypeId(testBackendName) + err = v.AttachRemoteDatFile(diskFile, &volume_server_pb.RemoteFile{ + BackendType: bType, BackendId: bId, Key: key, FileSize: uint64(size), + ModifiedTime: uint64(time.Now().Unix()), Extension: ".dat", + }) + var ndErr *volume_info.NotCrashDurableError + require.True(t, errors.As(err, &ndErr), "the fsync failure is still reported: %v", err) + + onDisk, _, _, err := volume_info.MaybeLoadVolumeInfo(v.FileName(".vif")) + require.NoError(t, err) + require.Len(t, onDisk.GetFiles(), 1) + require.Equal(t, key, onDisk.GetFiles()[0].Key) + require.Len(t, v.GetVolumeInfo().GetFiles(), 1, "the in-memory entry was rolled back while the .vif names the copy") + require.Equal(t, key, v.GetVolumeInfo().GetFiles()[0].Key) + require.True(t, v.HasRemoteFile()) + _, isDisk := v.DataBackend.(*backend.DiskFile) + require.False(t, isDisk, "the live backend must be the copy the .vif names") + + n := &needle.Needle{Id: types.Uint64ToNeedleId(3)} + _, err = v.readNeedle(n, &ReadOption{}, nil) + require.NoError(t, err) +} diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index f37ce0d1d7..12b86982fc 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -62,6 +62,14 @@ func (v *Volume) garbageLevel() float64 { return float64(deletedSize) / float64(fileSize) } +// errTieredVolumeCompact refuses a compaction of a volume whose .dat is in the +// remote tier: the copy reads the local .dat, which tiering removes (or leaves +// behind as a stale copy), and the reload after a commit would read the remote +// object through the compacted index. +func errTieredVolumeCompact(vid needle.VolumeId) error { + return fmt.Errorf("volume %d is tiered to remote storage, cannot compact", vid) +} + type CompactOptions struct { PreallocateBytes int64 MaxBytesPerSecond int64 @@ -87,6 +95,9 @@ func (v *Volume) CompactByVolumeData(opts *CompactOptions) error { return nil } glog.V(3).Infof("Compacting volume %d ...", v.Id) + if v.HasRemoteFile() { + return errTieredVolumeCompact(v.Id) + } //no need to lock for copy on write //v.accessLock.Lock() //defer v.accessLock.Unlock() @@ -132,6 +143,9 @@ func (v *Volume) CompactByIndex(opts *CompactOptions) error { return nil } glog.V(3).Infof("Compact2 volume %d ...", v.Id) + if v.HasRemoteFile() { + return errTieredVolumeCompact(v.Id) + } if !v.isCompactionInProgress.CompareAndSwap(false, true) { glog.V(0).Infof("Volume %d is already compacting2 ...", v.Id) @@ -183,6 +197,21 @@ func (v *Volume) CommitCompact() error { defer v.dataFileAccessLock.Unlock() glog.V(3).Infof("Got volume %d committing lock...", v.Id) + if v.HasRemoteFile() { + // The volume was tiered after the copy started: the reload would read + // the remote object through the compacted index. Drop the copy and its + // temporary needle map; no later vacuum of this volume would. + if v.tmpNm != nil { + v.tmpNm.Close() + v.tmpNm = nil + } + for _, ext := range []string{".cpd", ".cpx", ".cpldb"} { + if e := os.RemoveAll(v.FileName(ext)); e != nil { + glog.V(0).Infof("remove %s: %v", v.FileName(ext), e) + } + } + return errTieredVolumeCompact(v.Id) + } if v.nm != nil { v.nm.Close() v.nm = nil diff --git a/weed/storage/volume_vacuum_tiered_test.go b/weed/storage/volume_vacuum_tiered_test.go new file mode 100644 index 0000000000..e39de29385 --- /dev/null +++ b/weed/storage/volume_vacuum_tiered_test.go @@ -0,0 +1,111 @@ +package storage + +import ( + "bytes" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// A tiered volume that accepted deletes has garbage, but its .dat is remote: +// the compaction must be refused before any copy, and the store must not +// report the volume as a vacuum candidate. +func TestCompactRefusesTieredVolume(t *testing.T) { + b := newLocalDirBackend(t) + registerTestBackend(t, b) + dir := t.TempDir() + const vid = needle.VolumeId(34) + v, _ := tierUpVolumeLive(t, dir, vid, b) + defer v.Close() + require.True(t, v.HasRemoteFile()) + for i := 1; i <= 4; i++ { + _, err := v.deleteNeedle2(newRandomNeedle(uint64(i))) + require.NoError(t, err) + } + garbage := v.garbageLevel() + require.Greater(t, garbage, 0.0, "precondition: the deletes left garbage") + + require.ErrorContains(t, v.CompactByIndex(nil), "tiered") + require.ErrorContains(t, v.CompactByVolumeData(nil), "tiered") + require.False(t, util.FileExists(v.FileName(".cpd"))) + require.False(t, util.FileExists(v.FileName(".cpx"))) + + loc := &DiskLocation{Directory: dir, volumes: map[needle.VolumeId]*Volume{}} + loc.SetVolume(vid, v) + s := &Store{Locations: []*DiskLocation{loc}} + ratio, _, err := s.CheckCompactVolume(vid) + require.NoError(t, err) + // volume.tier.compact reads this ratio while the volume is still remote to + // decide whether to download and compact it; vacuum selection skips tiered + // volumes by their remote storage name instead. + require.Equal(t, garbage, ratio, "a tiered volume reports its real garbage ratio") +} + +// A volume tiered while its compaction copy ran (keeping the local .dat) must +// not commit: the reload would read the remote object, which has the old +// layout, through the compacted index. +func TestCommitCompactRefusesVolumeTieredDuringCopy(t *testing.T) { + t.Run("memory", func(t *testing.T) { testCommitCompactRefusesVolumeTieredDuringCopy(t, NeedleMapInMemory) }) + t.Run("leveldb", func(t *testing.T) { testCommitCompactRefusesVolumeTieredDuringCopy(t, NeedleMapLevelDb) }) +} + +func testCommitCompactRefusesVolumeTieredDuringCopy(t *testing.T, kind NeedleMapKind) { + b := newLocalDirBackend(t) + registerTestBackend(t, b) + dir := t.TempDir() + v, err := NewVolume(dir, dir, "", 35, kind, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + require.NoError(t, err) + defer v.Close() + + written := map[uint64][]byte{} + for i := uint64(1); i <= 6; i++ { + n := newRandomNeedle(i) + n.Cookie = 7 + _, _, _, err := v.writeNeedle2(n, true, false, false) + require.NoError(t, err) + written[i] = n.Data + } + for i := uint64(1); i <= 3; i++ { + _, err := v.deleteNeedle2(newEmptyNeedle(i)) + require.NoError(t, err) + } + require.NoError(t, v.CompactByIndex(nil)) + require.True(t, util.FileExists(v.FileName(".cpd"))) + require.NotNil(t, v.tmpNm, "precondition: the copy left its temporary needle map") + if kind == NeedleMapLevelDb { + require.True(t, util.FileExists(v.FileName(".cpldb")), "precondition: the copy left its temporary LevelDB") + } + + // tier the volume up, keeping the local .dat + diskFile, ok := v.DataBackend.(*backend.DiskFile) + require.True(t, ok) + key, size, err := b.CopyFile(diskFile.File, nil, 0) + require.NoError(t, err) + bType, bId := backend.BackendNameToTypeId(testBackendName) + v.GetVolumeInfo().Files = append(v.GetVolumeInfo().GetFiles(), &volume_server_pb.RemoteFile{ + BackendType: bType, BackendId: bId, Key: key, FileSize: uint64(size), + ModifiedTime: uint64(time.Now().Unix()), Extension: ".dat", + }) + require.NoError(t, v.SaveVolumeInfo()) + require.NoError(t, v.LoadRemoteFile()) + + require.ErrorContains(t, v.CommitCompact(), "tiered") + require.False(t, util.FileExists(v.FileName(".cpd"))) + require.False(t, util.FileExists(v.FileName(".cpx"))) + require.False(t, util.FileExists(v.FileName(".cpldb")), "the temporary LevelDB is left on disk") + require.Nil(t, v.tmpNm, "the temporary needle map is kept") + for i := uint64(4); i <= 6; i++ { + got := &needle.Needle{Id: types.Uint64ToNeedleId(i), Cookie: 7} + _, err := v.readNeedle(got, nil, nil) + require.NoError(t, err, "needle %d", i) + require.True(t, bytes.Equal(written[i], got.Data), "needle %d data", i) + } +} diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index 443c7bc814..a04fa746cc 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -235,6 +235,33 @@ func (v *Volume) syncWrite(n *needle.Needle, checkCookie bool, fsync bool) (offs if !isUnchanged { err = v.publishWriteRequest(n, offset, fsync) } + // The .idx row goes down before the write is acknowledged, same as the + // .dat record above; the access lock is dropped for the fsync the same + // way, since the row is already published. + v.dataFileAccessLock.Unlock() + if idxErr := v.syncIndex(); err == nil { + err = idxErr + } + v.dataFileAccessLock.Lock() + return +} + +// syncIndex flushes the .idx, the second half of a durable write: the row is +// already published and cannot be taken back, so a failed sync stops the +// volume taking writes rather than acknowledge an index entry that may not +// survive — the same split the Rust volume server makes between flush_dat +// and flush_idx. +func (v *Volume) syncIndex() (err error) { + if v.nm == nil { + return nil + } + if err = v.nm.Sync(); err != nil { + v.checkReadWriteError(err) + v.noWriteLock.Lock() + v.noWriteOrDelete = true + v.noWriteLock.Unlock() + err = fmt.Errorf("sync volume %d index: %w", v.Id, err) + } return } @@ -503,6 +530,7 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { stagedWriteOk := make([]bool, len(run)) stagedDeleteOffsets := make([]uint64, len(run)) stagedDeleteOk := make([]bool, len(run)) + needIndexSync := false for i := 0; i < len(run); i++ { // Every queued request meets the same refusal it would get sent on // its own: once a durable index update fails the volume stops taking @@ -514,9 +542,12 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { run[i].UpdateResult(0, 0, false, fmt.Errorf("volume %d is read only", v.Id)) } else { offset, size, isUnchanged, err := v.appendWriteRequest(run[i].N, true, true) - if err == nil && !isUnchanged { - stagedWriteOffsets[i] = offset - stagedWriteOk[i] = true + if err == nil { + needIndexSync = true + if !isUnchanged { + stagedWriteOffsets[i] = offset + stagedWriteOk[i] = true + } } run[i].UpdateResult(offset, uint64(size), isUnchanged, err) } @@ -527,9 +558,12 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { run[i].UpdateResult(0, 0, false, fmt.Errorf("volume %d is read only", v.Id)) } else { size, offset, tombstoned, err := v.appendDeleteRequest(run[i].N) - if err == nil && tombstoned { - stagedDeleteOffsets[i] = offset - stagedDeleteOk[i] = true + if err == nil { + needIndexSync = true + if tombstoned { + stagedDeleteOffsets[i] = offset + stagedDeleteOk[i] = true + } } run[i].UpdateResult(0, uint64(size), false, err) } @@ -588,6 +622,16 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { } } v.dataFileAccessLock.Unlock() + // One .idx sync per run, paired with the .dat sync above: the published + // rows cannot be rolled back, so a failed sync fails every request in the + // run instead of acknowledging an index entry that may not survive. + if needIndexSync { + if idxErr := v.syncIndex(); idxErr != nil { + for i := 0; i < len(run); i++ { + run[i].UpdateResult(0, 0, false, idxErr) + } + } + } for i := 0; i < len(run); i++ { run[i].Submit() } diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 7babdceb09..83e1ab6f47 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -39,6 +39,13 @@ func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid ne defer cancel() for index, dn := range locationlist.list { go func(index int, dn *DataNode, url pb.ServerAddress, vid needle.VolumeId) { + // A replica tiered to remote storage cannot be compacted in place, + // whatever its garbage ratio; volume.tier.compact reclaims it. + if v, lookErr := dn.GetVolumesById(vid); lookErr == nil && v.IsRemote() { + glog.V(1).Infof("skip vacuuming volume %d on %s: tiered to %s", vid, url, v.RemoteStorageName) + ch <- -1 + return + } err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { resp, err := volumeServerClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{ VolumeId: uint32(vid), diff --git a/weed/topology/topology_vacuum_test.go b/weed/topology/topology_vacuum_test.go index 6d96add773..82643f33b2 100644 --- a/weed/topology/topology_vacuum_test.go +++ b/weed/topology/topology_vacuum_test.go @@ -481,3 +481,52 @@ func TestVacuumStalledVolumeServerReleasesGuard(t *testing.T) { }) } } + +// A replica tiered to remote storage is never compacted by a vacuum, not even +// on an explicit request with a zero threshold: its .dat is the remote object. +// volume.tier.compact is the way to reclaim its garbage. +func TestVacuumSkipsTieredReplica(t *testing.T) { + fake := &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{ + 1: {GarbageRatio: 0.9}, + 2: {GarbageRatio: 0.9}, + }, + } + grpcPort, dialOption := startFakeVolumeServer(t, fake) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, grpcPort, "127.0.0.1", "dn", map[string]uint32{"": 10}) + newVolumeInfo := func(vid needle.VolumeId, remote string) storage.VolumeInfo { + return storage.VolumeInfo{ + Id: vid, + Size: 1 << 20, + Collection: "c", + ReadOnly: remote != "", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + RemoteStorageName: remote, + } + } + vl := topo.GetVolumeLayout("c", &super_block.ReplicaPlacement{}, needle.EMPTY_TTL, types.ToDiskType("")) + volumes := []storage.VolumeInfo{newVolumeInfo(1, "s3.default"), newVolumeInfo(2, "")} + dn.UpdateVolumes(volumes) + for _, v := range volumes { + topo.RegisterVolumeLayout(v, dn) + } + c := NewCollection("c", topo.volumeSizeLimit, false) + for _, vid := range []needle.VolumeId{1, 2} { + vl.accessLock.RLock() + ll := vl.vid2location[vid].Copy() + vl.accessLock.RUnlock() + topo.vacuumOneVolumeId(dialOption, vl, c, 0, ll, vid, 0, false) + } + + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.committed) != 1 || fake.committed[0] != 2 { + t.Fatalf("committed %v, want only the local volume 2", fake.committed) // volume 2 is the positive control + } +} diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.go index fba9225634..cf492230b0 100644 --- a/weed/util/http/http_global_client_util.go +++ b/weed/util/http/http_global_client_util.go @@ -119,6 +119,11 @@ func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error recordReachable(request.URL.Host) defer CloseResponse(response) + if response.StatusCode >= 400 { + retryable := response.StatusCode >= 500 + return nil, retryable, StatusError(url, response) + } + var reader io.ReadCloser switch response.Header.Get("Content-Encoding") { case "gzip": @@ -132,10 +137,6 @@ func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error } b, err := io.ReadAll(reader) - if response.StatusCode >= 400 { - retryable := response.StatusCode >= 500 - return nil, retryable, fmt.Errorf("%s: %s", url, response.Status) - } if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return nil, false, ctxErr @@ -334,7 +335,7 @@ func ReadUrl(ctx context.Context, fileUrl string, cipherKey []byte, isContentCom defer CloseResponse(r) if r.StatusCode >= 400 { - return 0, fmt.Errorf("%s: %s", fileUrl, r.Status) + return 0, StatusError(fileUrl, r) } var reader io.ReadCloser @@ -414,7 +415,7 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, return false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests) } retryable = r.StatusCode >= 499 - return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status) + return retryable, StatusError(fileUrl, r) } var reader io.ReadCloser @@ -504,8 +505,9 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt return nil, nil, err } if r.StatusCode >= 400 { + err = StatusError(fileUrl, r) CloseResponse(r) - return nil, nil, fmt.Errorf("%s: %s", fileUrl, r.Status) + return nil, nil, err } var reader io.ReadCloser @@ -523,6 +525,29 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt return r, reader, nil } +// StatusError adds the message a server writes into the body of a failed +// request, such as a volume server CRC mismatch, to the status line. +func StatusError(fileUrl string, r *http.Response) error { + var reader io.Reader = r.Body + if r.Header.Get("Content-Encoding") == "gzip" { + if gz, err := gzip.NewReader(r.Body); err == nil { + reader = gz + } + } + body, _ := io.ReadAll(io.LimitReader(reader, 1024)) + msg := strings.TrimSpace(string(body)) + var m struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &m) == nil && m.Error != "" { + msg = m.Error + } + if msg == "" { + return fmt.Errorf("%s: %s", fileUrl, r.Status) + } + return fmt.Errorf("%s: %s: %s", fileUrl, r.Status, msg) +} + func CloseResponse(resp *http.Response) { if resp == nil || resp.Body == nil { return @@ -782,7 +807,7 @@ func readUrlDirectToBuffer(ctx context.Context, fileUrl, jwt string, buffer []by return 0, false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests) } retryable = r.StatusCode >= 499 - return 0, retryable, fmt.Errorf("%s: %s", fileUrl, r.Status) + return 0, retryable, StatusError(fileUrl, r) } // Read directly into the buffer without intermediate copying diff --git a/weed/util/http/http_global_client_util_test.go b/weed/util/http/http_global_client_util_test.go index bd3d98d6c0..c5e5f32cc0 100644 --- a/weed/util/http/http_global_client_util_test.go +++ b/weed/util/http/http_global_client_util_test.go @@ -1,6 +1,8 @@ package http import ( + "bytes" + "compress/gzip" "context" "net/http" "net/http/httptest" @@ -92,6 +94,76 @@ func TestReadUrlAsStreamReturnsGzipReaderError(t *testing.T) { } } +func TestReadErrorsCarryTheServerMessage(t *testing.T) { + InitGlobalHttpClient() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"invalid CRC for needle 2a (got 0a1b2c3d, want 4e5f6071), data on disk corrupted: needle data corrupted"}`)) + })) + defer server.Close() + + want := server.URL + ": 500 Internal Server Error: invalid CRC for needle 2a (got 0a1b2c3d, want 4e5f6071), data on disk corrupted: needle data corrupted" + + if _, err := ReadUrl(context.Background(), server.URL, nil, false, true, 0, 0, make([]byte, 16)); err == nil || err.Error() != want { + t.Errorf("ReadUrl: got %v, want %q", err, want) + } + if _, err := ReadUrlAsStream(context.Background(), server.URL, "", nil, false, true, 0, 0, func(data []byte) {}); err == nil || err.Error() != want { + t.Errorf("ReadUrlAsStream: got %v, want %q", err, want) + } + if _, _, err := ReadUrlAsReaderCloser(server.URL, "", ""); err == nil || err.Error() != want { + t.Errorf("ReadUrlAsReaderCloser: got %v, want %q", err, want) + } + if _, _, err := readUrlDirectToBuffer(context.Background(), server.URL, "", make([]byte, 16)); err == nil || err.Error() != want { + t.Errorf("readUrlDirectToBuffer: got %v, want %q", err, want) + } + if _, _, err := GetAuthenticated(context.Background(), server.URL, ""); err == nil || err.Error() != want { + t.Errorf("GetAuthenticated: got %v, want %q", err, want) + } +} + +func TestReadErrorsDecodeGzipBody(t *testing.T) { + InitGlobalHttpClient() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = gz.Write([]byte(`{"error":"invalid CRC for needle 2a"}`)) + _ = gz.Close() + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write(buf.Bytes()) + })) + defer server.Close() + + want := server.URL + ": 500 Internal Server Error: invalid CRC for needle 2a" + if _, err := ReadUrl(context.Background(), server.URL, nil, false, true, 0, 0, make([]byte, 16)); err == nil || err.Error() != want { + t.Errorf("ReadUrl: got %v, want %q", err, want) + } + if _, err := ReadUrlAsStream(context.Background(), server.URL, "", nil, false, true, 0, 0, func(data []byte) {}); err == nil || err.Error() != want { + t.Errorf("ReadUrlAsStream: got %v, want %q", err, want) + } + if _, _, err := GetAuthenticated(context.Background(), server.URL, ""); err == nil || err.Error() != want { + t.Errorf("GetAuthenticated: got %v, want %q", err, want) + } +} + +func TestReadErrorWithoutBodyIsTheStatus(t *testing.T) { + InitGlobalHttpClient() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + want := server.URL + ": 500 Internal Server Error" + if _, err := ReadUrlAsStream(context.Background(), server.URL, "", nil, false, true, 0, 0, func(data []byte) {}); err == nil || err.Error() != want { + t.Errorf("got %v, want %q", err, want) + } +} + func TestDeleteReturnsInvalidRequestErrorBeforeAddingAuth(t *testing.T) { defer func() { if r := recover(); r != nil { diff --git a/weed/util/version/constants.go b/weed/util/version/constants.go index 987ee12425..852d163e40 100644 --- a/weed/util/version/constants.go +++ b/weed/util/version/constants.go @@ -9,7 +9,7 @@ import ( var ( MAJOR_VERSION = int32(4) - MINOR_VERSION = int32(48) + MINOR_VERSION = int32(49) VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION) VERSION = util.SizeLimit + " " + VERSION_NUMBER COMMIT = "" diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go index f2838ffc81..b9df6e3ed3 100644 --- a/weed/worker/tasks/erasure_coding/ec_task.go +++ b/weed/worker/tasks/erasure_coding/ec_task.go @@ -455,39 +455,60 @@ func (t *ErasureCodingTask) rollbackReadonly(_ context.Context) { } } +// ecCopyAttempts bounds how often the pinned file pair copy restarts after a +// concurrent compaction bumps the source's CompactionRevision mid-copy. +const ecCopyAttempts = 3 + // copyVolumeFilesToWorker copies .idx and .dat files from source server to local worker. // The .idx is copied first, then .dat. Both copies are capped to the sizes reported by -// ReadVolumeFileStatus. If a write lands after .idx is copied, .dat may include extra -// data not referenced by .idx (harmless). The reverse (idx referencing data past .dat) -// is caught by verifyDatIdxConsistency in generateEcShardsLocally. +// ReadVolumeFileStatus and pinned to its CompactionRevision so they come from the same +// volume generation. A concurrent vacuum bumps that revision mid-copy, so a compaction +// failure restarts the pair under a freshly read revision. func (t *ErasureCodingTask) copyVolumeFilesToWorker(ctx context.Context, workDir string) (map[string]string, error) { localFiles := make(map[string]string) - fileStatus, err := t.readSourceVolumeFileStatus(ctx) - if err != nil { - return nil, fmt.Errorf("failed to read source volume file status: %w", err) - } - t.expireAtSec = fileStatus.GetExpireAtSec() - if t.expireAtSec == 0 { - // Volume servers that predate the top-level field still carry the - // expiry in the nested .vif once SaveVolumeInfo has stamped it. - t.expireAtSec = fileStatus.GetVolumeInfo().GetExpireAtSec() - } + var copyErr error + for attempt := 0; attempt < ecCopyAttempts; attempt++ { + fileStatus, err := t.readSourceVolumeFileStatus(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read source volume file status: %w", err) + } + t.expireAtSec = fileStatus.GetExpireAtSec() + if t.expireAtSec == 0 { + // Volume servers that predate the top-level field still carry the + // expiry in the nested .vif once SaveVolumeInfo has stamped it. + t.expireAtSec = fileStatus.GetVolumeInfo().GetExpireAtSec() + } - t.GetLogger().WithFields(map[string]interface{}{ - "volume_id": t.volumeID, - "source": t.server, - "working_dir": workDir, - "compaction_revision": fileStatus.GetCompactionRevision(), - "dat_file_size_bytes": fileStatus.GetDatFileSize(), - "idx_file_size_bytes": fileStatus.GetIdxFileSize(), - }).Info("Starting volume file copy from source server") + t.GetLogger().WithFields(map[string]interface{}{ + "volume_id": t.volumeID, + "source": t.server, + "working_dir": workDir, + "compaction_revision": fileStatus.GetCompactionRevision(), + "dat_file_size_bytes": fileStatus.GetDatFileSize(), + "idx_file_size_bytes": fileStatus.GetIdxFileSize(), + }).Info("Starting volume file copy from source server") - // Copy .idx file FIRST — if a write lands on the source after this copy, - // the .dat copy will include the new data but .idx won't reference it. + if copyErr = t.copyVolumeFilePair(ctx, workDir, fileStatus, localFiles); copyErr == nil { + return localFiles, nil + } + if !isVolumeCompactedError(copyErr) || ctx.Err() != nil { + return nil, copyErr + } + glog.Warningf("volume %d on %s compacted during EC file copy, restarting with fresh file status: %v", t.volumeID, t.server, copyErr) + } + return nil, copyErr +} + +// copyVolumeFilePair copies .idx first, then .dat, pinned to the compaction +// revision in fileStatus. If a write lands after .idx is copied, .dat may +// include extra data not referenced by .idx (harmless). The reverse (idx +// referencing data past .dat) is caught by verifyDatIdxConsistency in +// generateEcShardsLocally. +func (t *ErasureCodingTask) copyVolumeFilePair(ctx context.Context, workDir string, fileStatus *volume_server_pb.ReadVolumeFileStatusResponse, localFiles map[string]string) error { idxFile := filepath.Join(workDir, fmt.Sprintf("%d.idx", t.volumeID)) if err := t.copyFileFromSource(ctx, ".idx", idxFile, fileStatus.GetCompactionRevision(), fileStatus.GetIdxFileSize()); err != nil { - return nil, fmt.Errorf("failed to copy .idx file: %w", err) + return fmt.Errorf("failed to copy .idx file: %w", err) } localFiles["idx"] = idxFile @@ -503,7 +524,7 @@ func (t *ErasureCodingTask) copyVolumeFilesToWorker(ctx context.Context, workDir // Copy .dat file SECOND — guaranteed to have at least as much data as .idx references. datFile := filepath.Join(workDir, fmt.Sprintf("%d.dat", t.volumeID)) if err := t.copyFileFromSource(ctx, ".dat", datFile, fileStatus.GetCompactionRevision(), fileStatus.GetDatFileSize()); err != nil { - return nil, fmt.Errorf("failed to copy .dat file: %w", err) + return fmt.Errorf("failed to copy .dat file: %w", err) } localFiles["dat"] = datFile @@ -516,7 +537,13 @@ func (t *ErasureCodingTask) copyVolumeFilesToWorker(ctx context.Context, workDir }).Info("Volume data file copied successfully") } - return localFiles, nil + return nil +} + +// isVolumeCompactedError matches the failure CopyFile returns when the source +// volume's CompactionRevision moved past the pinned one mid-copy. +func isVolumeCompactedError(err error) bool { + return err != nil && strings.Contains(err.Error(), "is compacted") } func (t *ErasureCodingTask) readSourceVolumeFileStatus(ctx context.Context) (*volume_server_pb.ReadVolumeFileStatusResponse, error) { diff --git a/weed/worker/tasks/erasure_coding/ec_task_copy_retry_test.go b/weed/worker/tasks/erasure_coding/ec_task_copy_retry_test.go new file mode 100644 index 0000000000..76294f6b14 --- /dev/null +++ b/weed/worker/tasks/erasure_coding/ec_task_copy_retry_test.go @@ -0,0 +1,166 @@ +package erasure_coding + +import ( + "context" + "fmt" + "net" + "os" + "sync" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// fakeCopySource is a stub VolumeServer. Rejecting a pinned .dat copy advances +// the compaction revision and rewrites the volume contents, so a mid-copy +// compaction is replayed deterministically. +type fakeCopySource struct { + volume_server_pb.UnimplementedVolumeServerServer + + mu sync.Mutex + revision uint32 + datBytes []byte + idxBytes []byte + expireAtSec uint64 + rejectDat int + failErr error + statusCalls int + copyCalls int +} + +func (f *fakeCopySource) snapshot() (revision uint32, expireAtSec uint64, statusCalls, copyCalls int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.revision, f.expireAtSec, f.statusCalls, f.copyCalls +} + +func (f *fakeCopySource) ReadVolumeFileStatus(context.Context, *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.statusCalls++ + return &volume_server_pb.ReadVolumeFileStatusResponse{ + CompactionRevision: f.revision, + DatFileSize: uint64(len(f.datBytes)), + IdxFileSize: uint64(len(f.idxBytes)), + ExpireAtSec: f.expireAtSec, + }, nil +} + +func (f *fakeCopySource) CopyFile(req *volume_server_pb.CopyFileRequest, stream grpc.ServerStreamingServer[volume_server_pb.CopyFileResponse]) error { + f.mu.Lock() + f.copyCalls++ + reject := req.GetExt() == ".dat" && f.rejectDat > 0 + if reject { + f.rejectDat-- + f.revision++ + f.datBytes = append(f.datBytes, 0xEC) + f.idxBytes = append(f.idxBytes, 0xEC) + f.expireAtSec += 100 + } + failErr := f.failErr + revision := f.revision + var content []byte + switch req.GetExt() { + case ".idx": + content = f.idxBytes + case ".dat": + content = f.datBytes + } + f.mu.Unlock() + + if failErr != nil { + return failErr + } + if reject || req.GetCompactionRevision() != revision { + return fmt.Errorf("volume %d is compacted", req.GetVolumeId()) + } + content = content[:req.GetStopOffset()] + for len(content) > 0 { + n := min(len(content), 4) + if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: content[:n]}); err != nil { + return err + } + content = content[n:] + } + return nil +} + +func startFakeCopySource(t *testing.T, src *fakeCopySource) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(grpcServer, src) + go grpcServer.Serve(lis) + t.Cleanup(grpcServer.Stop) + // server addresses carry httpPort.grpcPort; the gRPC dialer uses the part + // after the dot, so report the listening port that way + _, port, err := net.SplitHostPort(lis.Addr().String()) + require.NoError(t, err) + return "127.0.0.1:0." + port +} + +func newCopyRetryTask(addr string) *ErasureCodingTask { + return NewErasureCodingTask("copy-retry", addr, 951, "", grpc.WithTransportCredentials(insecure.NewCredentials())) +} + +func TestCopyVolumeFilesToWorkerRetriesAfterMidCopyCompaction(t *testing.T) { + src := &fakeCopySource{ + revision: 7, + idxBytes: []byte("idx-data-v7"), + datBytes: []byte("dat-payload-v7"), + expireAtSec: 1000, + rejectDat: 1, + } + task := newCopyRetryTask(startFakeCopySource(t, src)) + + localFiles, err := task.copyVolumeFilesToWorker(context.Background(), t.TempDir()) + require.NoError(t, err) + + datContent, err := os.ReadFile(localFiles["dat"]) + require.NoError(t, err) + require.Equal(t, []byte("dat-payload-v7\xEC"), datContent) + idxContent, err := os.ReadFile(localFiles["idx"]) + require.NoError(t, err) + require.Equal(t, []byte("idx-data-v7\xEC"), idxContent) + revision, expireAtSec, statusCalls, copyCalls := src.snapshot() + require.Equal(t, uint32(8), revision) + require.Equal(t, 2, statusCalls) + require.Equal(t, 4, copyCalls) + require.Equal(t, expireAtSec, task.expireAtSec) +} + +func TestCopyVolumeFilesToWorkerStopsAfterRetryLimit(t *testing.T) { + src := &fakeCopySource{ + revision: 1, + idxBytes: []byte("i"), + datBytes: []byte("d"), + rejectDat: 10, + } + task := newCopyRetryTask(startFakeCopySource(t, src)) + + _, err := task.copyVolumeFilesToWorker(context.Background(), t.TempDir()) + require.ErrorContains(t, err, "is compacted") + _, _, statusCalls, copyCalls := src.snapshot() + require.Equal(t, ecCopyAttempts, statusCalls) + require.Equal(t, 2*ecCopyAttempts, copyCalls) +} + +func TestCopyVolumeFilesToWorkerReturnsNonCompactionErrors(t *testing.T) { + src := &fakeCopySource{ + revision: 1, + idxBytes: []byte("i"), + datBytes: []byte("d"), + failErr: fmt.Errorf("boom"), + } + task := newCopyRetryTask(startFakeCopySource(t, src)) + + _, err := task.copyVolumeFilesToWorker(context.Background(), t.TempDir()) + require.ErrorContains(t, err, "boom") + _, _, statusCalls, copyCalls := src.snapshot() + require.Equal(t, 1, statusCalls) + require.Equal(t, 1, copyCalls) +} diff --git a/weed/worker/tasks/iceberg/compact_order.go b/weed/worker/tasks/iceberg/compact_order.go index 56b6176fbb..ffda8acc73 100644 --- a/weed/worker/tasks/iceberg/compact_order.go +++ b/weed/worker/tasks/iceberg/compact_order.go @@ -37,7 +37,7 @@ func resolveCompactionOrder(meta table.Metadata, entries []iceberg.ManifestEntry continue } typ, ok := field.Type.(iceberg.PrimitiveType) - if !ok { + if !ok || !orderableType(typ) { continue } candidate := &compactionOrder{fieldID: field.ID, typ: typ, descending: sortField.Direction == table.SortDESC} @@ -57,7 +57,7 @@ func resolveCompactionOrder(meta table.Metadata, entries []iceberg.ManifestEntry // different format, or another partition) cannot disable ordering here. for _, field := range schema.Fields() { typ, ok := field.Type.(iceberg.PrimitiveType) - if !ok { + if !ok || !orderableType(typ) { continue } candidate := &compactionOrder{fieldID: field.ID, typ: typ, bestEffort: true} @@ -68,6 +68,18 @@ func resolveCompactionOrder(meta table.Metadata, entries []iceberg.ManifestEntry return nil } +// orderableType reports whether bounds of the type order files. Geometry and +// geography bounds are corners of a box: they decode into literals that do +// not compare, so ordering by them would sort nothing while still splitting +// oversized bins as if the files were in order. +func orderableType(t iceberg.PrimitiveType) bool { + switch t.(type) { + case iceberg.GeometryType, iceberg.GeographyType: + return false + } + return true +} + func (o *compactionOrder) coversEntries(entries []iceberg.ManifestEntry) bool { for _, entry := range entries { if _, ok := o.key(entry); !ok { diff --git a/weed/worker/tasks/iceberg/compact_order_test.go b/weed/worker/tasks/iceberg/compact_order_test.go index 8a4b1fdf78..1a36f7618f 100644 --- a/weed/worker/tasks/iceberg/compact_order_test.go +++ b/weed/worker/tasks/iceberg/compact_order_test.go @@ -386,3 +386,36 @@ func TestCompactDataFilesKeepsRunsWhenRepackFails(t *testing.T) { t.Fatalf("expected the ordered run to survive the failed repack, got %q", result) } } + +// Geometry bounds have no order, so they never order a bin's files: a table +// whose first column is a geometry is ordered by the next column with bounds. +func TestResolveCompactionOrderSkipsGeometry(t *testing.T) { + schema := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Type: iceberg.GeometryType{}, Name: "geom"}, + iceberg.NestedField{ID: 2, Type: iceberg.PrimitiveTypes.Int64, Name: "id", Required: true}) + meta, err := table.NewMetadata(schema, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, "s3://tb/ns/geo", iceberg.Properties{"format-version": "3"}) + if err != nil { + t.Fatal(err) + } + geomField, _ := meta.CurrentSchema().FindFieldByName("geom") + idField, _ := meta.CurrentSchema().FindFieldByName("id") + snap := int64(1) + var entries []iceberg.ManifestEntry + for i, name := range []string{"a.parquet", "b.parquet", "c.parquet"} { + b, err := iceberg.NewDataFileBuilder(*iceberg.UnpartitionedSpec, iceberg.EntryContentData, "s3://tb/ns/geo/data/"+name, iceberg.ParquetFile, map[int]any{}, nil, nil, 1, 100) + if err != nil { + t.Fatal(err) + } + b.LowerBoundValues(map[int][]byte{geomField.ID: xyBound(float64(i), 0), idField.ID: geoLE64(int64(10 - i))}) + b.UpperBoundValues(map[int][]byte{geomField.ID: xyBound(float64(i)+1, 1), idField.ID: geoLE64(int64(10 - i))}) + entries = append(entries, iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snap, nil, nil, b.Build())) + } + order := resolveCompactionOrder(meta, entries) + if order == nil || order.fieldID != idField.ID { + t.Fatalf("order %+v, want the id column (field %d), not the geometry (field %d)", order, idField.ID, geomField.ID) + } + order.sortEntries(entries) + if first := entries[0].DataFile().FilePath(); !strings.HasSuffix(first, "c.parquet") { + t.Errorf("first file %s, want c.parquet (the lowest id)", first) + } +} diff --git a/weed/worker/tasks/iceberg/compact_stats.go b/weed/worker/tasks/iceberg/compact_stats.go index 5ee092eeca..277752c976 100644 --- a/weed/worker/tasks/iceberg/compact_stats.go +++ b/weed/worker/tasks/iceberg/compact_stats.go @@ -64,6 +64,7 @@ func collectColumnStats(data []byte, schema *iceberg.Schema) (*columnStats, erro stats := &columnStats{sizes: map[int]int64{}, values: map[int]int64{}, nulls: map[int]int64{}, lower: map[int][]byte{}, upper: map[int][]byte{}} agg := map[int]*bounds{} + geoAgg := map[int]*geometryBounds{} meta := file.Metadata() for rgIdx, rg := range file.RowGroups() { stats.splitOffsets = append(stats.splitOffsets, rowGroupOffset(&meta.RowGroups[rgIdx])) @@ -82,9 +83,24 @@ func collectColumnStats(data []byte, schema *iceberg.Schema) (*columnStats, erro chunk, ok := cc.(*parquet.FileColumnChunk) if !ok { b.invalid = true + geoAgg[field.ID] = &geometryBounds{invalid: true} continue } stats.nulls[field.ID] += chunk.NullCount() + switch field.Type.(type) { + case iceberg.GeometryType: + // the bound comes from the row group's geospatial statistics + g := geoAgg[field.ID] + if g == nil { + g = &geometryBounds{} + geoAgg[field.ID] = g + } + g.add(meta.RowGroups[rgIdx].Columns[col].MetaData.GeospatialStatistics, chunk.NumValues(), chunk.NullCount()) + continue + case iceberg.GeographyType: + // no bound: edges are geodesic and a box may cross the antimeridian + continue + } if chunk.NullCount() == chunk.NumValues() { continue // an all-null chunk constrains nothing } @@ -122,6 +138,11 @@ func collectColumnStats(data []byte, schema *iceberg.Schema) (*columnStats, erro stats.upper[id] = raw } } + for id, g := range geoAgg { + if lower, upper, ok := g.encoded(); ok { + stats.lower[id], stats.upper[id] = lower, upper + } + } sort.Slice(stats.splitOffsets, func(i, j int) bool { return stats.splitOffsets[i] < stats.splitOffsets[j] }) return stats, nil } diff --git a/weed/worker/tasks/iceberg/geometry_bounds.go b/weed/worker/tasks/iceberg/geometry_bounds.go new file mode 100644 index 0000000000..34789c7d2d --- /dev/null +++ b/weed/worker/tasks/iceberg/geometry_bounds.go @@ -0,0 +1,71 @@ +package iceberg + +import ( + "encoding/binary" + "math" + + "github.com/parquet-go/parquet-go/format" +) + +// geometryBounds aggregates the bound of a geometry column over the row groups +// of a written file. The spec serializes a geometry bound as a point: the +// lower bound is (xmin, ymin), the upper bound (xmax, ymax), each coordinate +// an 8-byte little-endian double; z and m are optional and not written here. +// +// Readers skip a file whose bound does not intersect a spatial filter, so a +// box smaller than the file's geometries returns wrong results. Any doubt +// gives no bound: +// - the box is the union of the row groups' geospatial statistics, which +// the Parquet writer computes from every non-null value; +// - a row group with non-null values but no statistics (the writer omits +// them when a value does not parse as WKB) gives the column no bound, as +// does a box that is not finite or is inverted; +// - a row group of nulls only constrains nothing. +type geometryBounds struct { + xmin, ymin, xmax, ymax float64 + seen bool + invalid bool +} + +func (g *geometryBounds) add(gs format.GeospatialStatistics, numValues, nullCount int64) { + if g.invalid || numValues <= 0 || nullCount >= numValues { + return + } + if len(gs.GeoSpatialTypes) == 0 { + g.invalid = true + return + } + bb := gs.BBox + for _, v := range []float64{bb.XMin, bb.XMax, bb.YMin, bb.YMax} { + if math.IsNaN(v) || math.IsInf(v, 0) { + g.invalid = true + return + } + } + if bb.XMin > bb.XMax || bb.YMin > bb.YMax { + g.invalid = true + return + } + if !g.seen { + g.xmin, g.ymin, g.xmax, g.ymax, g.seen = bb.XMin, bb.YMin, bb.XMax, bb.YMax, true + return + } + g.xmin, g.ymin = math.Min(g.xmin, bb.XMin), math.Min(g.ymin, bb.YMin) + g.xmax, g.ymax = math.Max(g.xmax, bb.XMax), math.Max(g.ymax, bb.YMax) +} + +// encoded returns the lower and upper bound, or false when the column gets +// none. +func (g *geometryBounds) encoded() (lower, upper []byte, ok bool) { + if g == nil || g.invalid || !g.seen { + return nil, nil, false + } + return encodeGeometryPoint(g.xmin, g.ymin), encodeGeometryPoint(g.xmax, g.ymax), true +} + +func encodeGeometryPoint(x, y float64) []byte { + b := make([]byte, 16) + binary.LittleEndian.PutUint64(b, math.Float64bits(x)) + binary.LittleEndian.PutUint64(b[8:], math.Float64bits(y)) + return b +} diff --git a/weed/worker/tasks/iceberg/geometry_bounds_test.go b/weed/worker/tasks/iceberg/geometry_bounds_test.go new file mode 100644 index 0000000000..dea73f4ce8 --- /dev/null +++ b/weed/worker/tasks/iceberg/geometry_bounds_test.go @@ -0,0 +1,215 @@ +package iceberg + +import ( + "fmt" + "math" + "strings" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/parquet-go/parquet-go/format" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +type geoBox struct{ xmin, ymin, xmax, ymax float64 } + +func (b geoBox) String() string { + return fmt.Sprintf("(%g %g, %g %g)", b.xmin, b.ymin, b.xmax, b.ymax) +} + +func (b geoBox) covers(o geoBox) bool { + return b.xmin <= o.xmin && b.ymin <= o.ymin && b.xmax >= o.xmax && b.ymax >= o.ymax +} + +// compactedGeoBounds is the geometry bound of each compacted file in the +// current manifests (nil: none), and whether its id column has a bound, the +// control that statistics were collected at all. +func compactedGeoBounds(t *testing.T, client filer_pb.SeaweedFilerClient, setup tableSetup) (map[string]*geoBox, map[string]bool) { + t.Helper() + geo, idBounded := map[string]*geoBox{}, map[string]bool{} + for _, e := range currentDataEntries(t, client, setup) { + df := e.DataFile() + if !strings.Contains(df.FilePath(), "/compact-") { + continue + } + idBounded[df.FilePath()] = df.LowerBoundValues()[geoTestIDField] != nil + lo, hi := df.LowerBoundValues()[geoTestGeomField], df.UpperBoundValues()[geoTestGeomField] + if lo == nil && hi == nil { + geo[df.FilePath()] = nil + continue + } + x0, y0 := decodeXYBound(t, lo) + x1, y1 := decodeXYBound(t, hi) + geo[df.FilePath()] = &geoBox{x0, y0, x1, y1} + } + if len(geo) == 0 { + t.Fatal("no compacted file in the manifests") + } + return geo, idBounded +} + +// mixedGeoFiles holds points and polygons far apart, so a bound that misses +// any of them shows. +func mixedGeoFiles() ([]geoFile, geoBox) { + files := []geoFile{ + {Name: "pts.parquet", Rows: []geoRow{{1, wkbPoint(-3.70, 40.41)}, {2, wkbPoint(2.17, 41.38)}, {3, wkbPoint(-5.98, 37.38)}}}, + {Name: "boxes.parquet", Rows: []geoRow{{4, wkbBox(-9.3, 42.1, -8.5, 43.4)}, {5, wkbBox(1.0, 36.0, 4.3, 39.9)}}}, + {Name: "nulls.parquet", Rows: []geoRow{{6, nil}, {7, nil}}}, + {Name: "mixed.parquet", Rows: []geoRow{{8, nil}, {9, wkbPoint(-1.10, 40.34)}, {10, wkbBox(-0.5, 38.9, -0.3, 39.5)}}}, + } + return files, geoBox{-9.3, 36.0, 4.3, 43.4} +} + +func TestCompactRecordsGeometryBounds(t *testing.T) { + fs, client := startFakeFiler(t) + setup := tableSetup{BucketName: "tb", Namespace: "ns", TableName: "geo"} + files, want := mixedGeoFiles() + populateGeoTable(t, fs, setup, files, geoTableOptions{}) + if result := compactGeo(t, client, setup, Config{}); !strings.Contains(result, "compacted 4 files") { + t.Fatalf("result %q", result) + } + geo, idBounded := compactedGeoBounds(t, client, setup) + for p, got := range geo { + if !idBounded[p] { + t.Errorf("%s: no id bound: statistics were not collected", p) + } + if got == nil { + t.Fatalf("%s: no geometry bound", p) + } + if *got != want { + t.Errorf("%s: geometry bound %s, want the union of every geometry %s", p, got, want) + } + } +} + +// A sorted rewrite may write its output without row-group geospatial +// statistics; the file then gets no geometry bound, never one that misses +// a row. +func TestCompactSortedGeometryBoundCoversEveryRow(t *testing.T) { + fs, client := startFakeFiler(t) + setup := tableSetup{BucketName: "tb", Namespace: "ns", TableName: "geo"} + so, err := table.NewSortOrder(1, []table.SortField{{SourceIDs: []int{geoTestIDField}, Transform: iceberg.IdentityTransform{}, Direction: table.SortDESC, NullOrder: table.NullsFirst}}) + if err != nil { + t.Fatal(err) + } + files, want := mixedGeoFiles() + populateGeoTable(t, fs, setup, files, geoTableOptions{sortOrder: &so}) + compactGeo(t, client, setup, Config{RewriteStrategy: "sort", SortBufferRows: minSortBufferRows}) + geo, _ := compactedGeoBounds(t, client, setup) + for p, got := range geo { + if got != nil && !got.covers(want) { + t.Errorf("%s: geometry bound %s does not cover every geometry %s", p, got, want) + } + } +} + +func TestCompactAllNullGeometryGetsNoBound(t *testing.T) { + fs, client := startFakeFiler(t) + setup := tableSetup{BucketName: "tb", Namespace: "ns", TableName: "geo"} + populateGeoTable(t, fs, setup, []geoFile{ + {Name: "a.parquet", Rows: []geoRow{{1, nil}, {2, nil}}}, + {Name: "b.parquet", Rows: []geoRow{{3, nil}}}, + }, geoTableOptions{}) + compactGeo(t, client, setup, Config{}) + geo, idBounded := compactedGeoBounds(t, client, setup) + for p, got := range geo { + if got != nil { + t.Errorf("%s: geometry bound %s on a column of nulls", p, got) + } + if !idBounded[p] { + t.Errorf("%s: no id bound: statistics were not collected", p) + } + } +} + +// A value that does not parse as WKB leaves its row group without +// geospatial statistics; the file then gets no geometry bound rather than +// one from the other row groups that does not cover that row group. +func TestCompactUnparseableGeometryGivesNoBound(t *testing.T) { + fs, client := startFakeFiler(t) + setup := tableSetup{BucketName: "tb", Namespace: "ns", TableName: "geo"} + populateGeoTable(t, fs, setup, []geoFile{ + {Name: "ok.parquet", Rows: []geoRow{{1, wkbPoint(10, 10)}, {2, wkbPoint(11, 11)}}}, + {Name: "bad.parquet", Rows: []geoRow{{3, []byte("not wkb at all")}, {4, wkbPoint(-50, -50)}}}, + }, geoTableOptions{}) + // two rows a row group: the valid file's rows, then the other file's + compactGeo(t, client, setup, Config{RowGroupRowLimit: 2}) + geo, idBounded := compactedGeoBounds(t, client, setup) + for p, got := range geo { + if got != nil { + t.Errorf("%s: geometry bound %s although a value did not parse", p, got) + } + if !idBounded[p] { + t.Errorf("%s: no id bound: statistics were not collected", p) + } + } +} + +func TestCollectColumnStatsGivesGeographyNoBound(t *testing.T) { + data := writeGeoParquet(t, []geoRow{{1, wkbPoint(170, 10)}, {2, wkbPoint(-170, 12)}}, "") + geog, err := iceberg.GeographyTypeOf(iceberg.DefaultGeoCRS, "spherical") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + typ iceberg.Type + wantBound bool + }{{iceberg.GeometryType{}, true}, {geog, false}} { + stats, err := collectColumnStats(data, geoTestSchema(tc.typ)) + if err != nil { + t.Fatal(err) + } + if _, has := stats.lower[geoTestGeomField]; has != tc.wantBound { + t.Errorf("%s: lower bound present %v, want %v", tc.typ, has, tc.wantBound) + } + if stats.values[geoTestGeomField] != 2 { + t.Errorf("%s: value count %d, want 2", tc.typ, stats.values[geoTestGeomField]) + } + } +} + +func TestGeometryBoundsAggregation(t *testing.T) { + stats := func(xmin, ymin, xmax, ymax float64) format.GeospatialStatistics { + return format.GeospatialStatistics{BBox: format.BoundingBox{XMin: xmin, XMax: xmax, YMin: ymin, YMax: ymax}, GeoSpatialTypes: []int32{1}} + } + type chunk struct { + gs format.GeospatialStatistics + values, nulls int64 + } + for _, tc := range []struct { + name string + chunks []chunk + want *geoBox + }{ + {"one", []chunk{{stats(1, 2, 3, 4), 5, 0}}, &geoBox{1, 2, 3, 4}}, + {"union", []chunk{{stats(1, 2, 3, 4), 5, 0}, {stats(-1, 3, 2, 9), 5, 1}}, &geoBox{-1, 2, 3, 9}}, + {"nulls only constrain nothing", []chunk{{stats(1, 2, 3, 4), 5, 0}, {format.GeospatialStatistics{}, 4, 4}}, &geoBox{1, 2, 3, 4}}, + {"values without statistics", []chunk{{stats(1, 2, 3, 4), 5, 0}, {format.GeospatialStatistics{}, 4, 1}}, nil}, + {"NaN", []chunk{{stats(1, math.NaN(), 3, 4), 5, 0}}, nil}, + {"infinity", []chunk{{stats(1, 2, math.Inf(1), 4), 5, 0}}, nil}, + {"inverted", []chunk{{stats(3, 2, 1, 4), 5, 0}}, nil}, + {"nothing", nil, nil}, + } { + var g geometryBounds + for _, c := range tc.chunks { + g.add(c.gs, c.values, c.nulls) + } + lo, hi, ok := g.encoded() + if tc.want == nil { + if ok { + t.Errorf("%s: bound given, want none", tc.name) + } + continue + } + if !ok { + t.Errorf("%s: no bound, want %s", tc.name, tc.want) + continue + } + x0, y0 := decodeXYBound(t, lo) + x1, y1 := decodeXYBound(t, hi) + if got := (geoBox{x0, y0, x1, y1}); got != *tc.want { + t.Errorf("%s: bound %s, want %s", tc.name, got, tc.want) + } + } +} diff --git a/weed/worker/tasks/vacuum/detection.go b/weed/worker/tasks/vacuum/detection.go index a52cca18f6..bc5eb41d9e 100644 --- a/weed/worker/tasks/vacuum/detection.go +++ b/weed/worker/tasks/vacuum/detection.go @@ -109,10 +109,14 @@ func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterI } // pickPrimaryReplica returns the eligible replica with the highest garbage -// ratio. Returns nil if no replica satisfies both gates. +// ratio. Returns nil if no replica satisfies both gates. Replicas tiered to +// remote storage are skipped: the volume server cannot compact them. func pickPrimaryReplica(replicas []*types.VolumeHealthMetrics, garbageThreshold float64, minAge time.Duration) *types.VolumeHealthMetrics { var best *types.VolumeHealthMetrics for _, m := range replicas { + if m.HasRemoteCopy { + continue + } if m.GarbageRatio < garbageThreshold { continue } @@ -144,7 +148,7 @@ func recordSkipReasons(replicas []*types.VolumeHealthMetrics, garbageThreshold f } // createVacuumTaskParams creates typed parameters for a vacuum task whose -// Sources list contains every replica of the volume. Worker-side +// Sources list contains every non-tiered replica of the volume. Worker-side // performVacuum iterates over Sources and vacuums each replica. func createVacuumTaskParams(task *types.TaskDetectionResult, replicas []*types.VolumeHealthMetrics, vacuumConfig *Config, clusterInfo *types.ClusterInfo) *worker_pb.TaskParams { garbageThreshold := 0.3 @@ -164,6 +168,9 @@ func createVacuumTaskParams(task *types.TaskDetectionResult, replicas []*types.V sources := make([]*worker_pb.TaskSource, 0, len(replicas)) seen := make(map[string]struct{}, len(replicas)) for _, m := range replicas { + if m.HasRemoteCopy { + continue + } address, err := util.ResolveServerAddress(m.Server, clusterInfo.ActiveTopology) if err != nil { glog.Warningf("Failed to resolve address for server %s for vacuum task on volume %d, dropping replica: %v", m.Server, task.VolumeID, err) diff --git a/weed/worker/tasks/vacuum/detection_tiered_test.go b/weed/worker/tasks/vacuum/detection_tiered_test.go new file mode 100644 index 0000000000..5232b45ea9 --- /dev/null +++ b/weed/worker/tasks/vacuum/detection_tiered_test.go @@ -0,0 +1,25 @@ +package vacuum + +import ( + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/worker/types" +) + +func TestPickPrimaryReplicaSkipsTieredReplicas(t *testing.T) { + m := func(id uint32, server string, remote bool) *types.VolumeHealthMetrics { + return &types.VolumeHealthMetrics{VolumeID: id, Server: server, GarbageRatio: 0.9, Age: 72 * time.Hour, + HasRemoteCopy: remote, IsReadOnly: remote} + } + if p := pickPrimaryReplica([]*types.VolumeHealthMetrics{m(1, "a:8080", false)}, 0.3, 24*time.Hour); p == nil { + t.Fatal("a local volume at 90% garbage must be picked") + } + if p := pickPrimaryReplica([]*types.VolumeHealthMetrics{m(2, "a:8080", true)}, 0.3, 24*time.Hour); p != nil { + t.Fatalf("tiered replica of volume %d picked for vacuum", p.VolumeID) + } + p := pickPrimaryReplica([]*types.VolumeHealthMetrics{m(3, "a:8080", true), m(3, "b:8080", false)}, 0.3, 24*time.Hour) + if p == nil || p.Server != "b:8080" { + t.Fatalf("partially tiered volume should pick its local replica, got %+v", p) + } +}