Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-10-23 22:40:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/seaweedfs/seaweedfs into nfs-enable-inode-index

Browse Source
This commit is contained in:
Chris Lu committed 2026-10-23 01:16:00 -07:00
2 parents 8311221fb4 b65832bdc9
commit 347d02cf9e
213 files changed
+16701 -1571
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/posix-compliance.yml
+777
View File
@@ -0,0 +1,777 @@
name: POSIX Compliance Tests
on:
push:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
pull_request:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
workflow_dispatch:
inputs:
test_type:
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
required: true
default: 'full'
type: choice
options:
- critical
- basic
- extended
- full
enable_external_tests:
description: 'Run external test suites (slower)'
required: false
default: false
type: boolean
permissions:
contents: read
pull-requests: write
issues: write
env:
GO_VERSION: '1.26'
TIMEOUT: '45m'
jobs:
posix-compliance-ubuntu:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
fuse-version: ['2.9', '3.0']
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
fetch-depth: 1
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: true
cache-dependency-path: |
go.sum
test/fuse_integration/go.sum
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
attr \
acl \
build-essential \
git \
python3-pip
# Install FUSE version specific packages
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
echo "Installing FUSE 2.9..."
sudo apt-get install -y fuse libfuse-dev
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
echo "Installing FUSE 3.0..."
sudo apt-get install -y fuse3 libfuse3-dev
# Also install fuse2 for compatibility if needed
sudo apt-get install -y fuse libfuse-dev
fi
- name: Set up user permissions for FUSE
run: |
# Create fuse group if it doesn't exist
sudo groupadd -f fuse
# Add user to fuse group
sudo usermod -a -G fuse $USER
# Set permissions on FUSE device
sudo chmod 666 /dev/fuse
# Ensure fuse module is loaded
sudo modprobe fuse || true
# Verify setup
echo "FUSE setup verification:"
ls -la /dev/fuse
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
lsmod | grep fuse || echo "FUSE module not loaded"
- name: Install external test tools
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
run: |
# Install nfstest for POSIX API verification
pip3 install --user nfstest
# Install FIO for performance testing
sudo apt-get install -y fio
# Install additional test utilities
sudo apt-get install -y stress-ng
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
# Make weed binary available in PATH (it's already there from go install)
sudo cp $(which weed) /usr/local/bin/weed
which weed
weed version
- name: Set up SeaweedFS cluster
run: |
start_and_wait() {
local name=$1
local url=$2
local pidfile=$3
shift 3
local cmd="$@"
echo "Starting $name..."
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
local pid=$!
echo $pid > "$pidfile"
echo "Waiting for $name to start..."
for i in {1..30}; do
if curl -sf "$url" > /dev/null 2>&1; then
echo "$name is ready"
return 0
fi
if [ $i -eq 30 ]; then
echo "$name failed to start"
cat "/tmp/seaweedfs/$name.log"
exit 1
fi
sleep 2
done
}
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
echo "SeaweedFS cluster status:"
curl -s http://127.0.0.1:9333/cluster/status || true
- name: Set up FUSE mount
run: |
# Create mount point
MOUNT_POINT="/tmp/seaweedfs/mount"
mkdir -p $MOUNT_POINT
echo "Mounting SeaweedFS FUSE filesystem..."
# Mount SeaweedFS FUSE filesystem in background
weed mount \
-filer=127.0.0.1:8888 \
-dir=$MOUNT_POINT \
-filer.path=/ \
-dirAutoCreate=true \
-allowOthers=true \
-nonempty=true \
> /tmp/seaweedfs/mount.log 2>&1 &
MOUNT_PID=$!
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
# Wait for mount to be ready. The mount point directory already
# exists, so only a real mount counts — a directory listing would
# pass on the local disk even if weed mount died.
echo "Waiting for FUSE mount to be ready..."
for i in {1..30}; do
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "FUSE mount is ready"
break
fi
if ! kill -0 $MOUNT_PID 2>/dev/null; then
echo "FUSE mount process exited"
cat /tmp/seaweedfs/mount.log
exit 1
fi
if [ $i -eq 30 ]; then
echo "FUSE mount failed to be ready"
cat /tmp/seaweedfs/mount.log
echo "Mount status:"
mount | grep fuse || echo "No FUSE mounts found"
ps aux | grep weed || echo "No weed processes found"
exit 1
fi
sleep 2
done
# Verify mount is working
echo "Testing FUSE mount functionality..."
echo "test" > $MOUNT_POINT/test_file.txt
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
echo "FUSE mount is working correctly"
rm $MOUNT_POINT/test_file.txt
else
echo "FUSE mount is not working properly"
exit 1
fi
# Export mount point for tests
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
- name: Set up test environment
run: |
cd test/fuse_integration
# Initialize Go module for tests
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
# Create reports directory
mkdir -p reports
# Set up external tools if requested
# Note: External tools setup is now optional and won't fail the build
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
echo "Setting up external POSIX testing tools..."
make -f posix_Makefile setup-external-tools || true
# setup-external-tools exits 0 even when a sub-target failed, so
# require the tools only when they are actually installed
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
echo "✅ External tools setup completed successfully"
else
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
echo "External tools will be skipped but tests will continue"
echo "This is expected behavior when build tools are not available"
fi
fi
# Verify SeaweedFS cluster is accessible
echo "Verifying SeaweedFS cluster accessibility..."
curl -sf http://127.0.0.1:9333/cluster/status
curl -sf http://127.0.0.1:8080/status
curl -sf http://127.0.0.1:8888/
# Verify mount point
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
ls -la $SEAWEEDFS_MOUNT_POINT
# Verify mount is actually working by testing file operations
echo "Verifying SeaweedFS mount functionality..."
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
echo "SeaweedFS mount test" > "$TEST_FILE"
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
echo "✅ Mount verification successful"
rm -f "$TEST_FILE"
else
echo "❌ Mount verification failed - mount may not be working properly"
exit 1
fi
# Check if this is actually a FUSE mount — the plain directory created
# above passes the file test, so without this the suite can run
# against the local disk and report a pass it never earned
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
echo "✅ FUSE mount detected:"
mount | grep "$SEAWEEDFS_MOUNT_POINT"
else
echo "❌ Mount point not found in mount table - refusing to test a local directory"
exit 1
fi
- name: Run POSIX compliance tests
id: posix-tests
env:
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
run: |
cd test/fuse_integration
# Determine which tests to run
TEST_TYPE="${{ github.event.inputs.test_type }}"
if [ -z "$TEST_TYPE" ]; then
TEST_TYPE="full"
fi
echo "Running POSIX tests: $TEST_TYPE"
if [ "$TEST_TYPE" = "full" ]; then
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
else
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
fi
# Set test configuration to use our mounted filesystem
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Debug: Show environment variables
echo "🔍 Test Environment:"
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
case "$TEST_TYPE" in
"critical")
echo "Running critical POSIX tests (basic compliance only)..."
make -f posix_Makefile test-posix-critical
;;
"basic")
echo "Running basic POSIX tests (core file operations)..."
make -f posix_Makefile test-posix-basic
;;
"extended")
echo "Running extended POSIX tests (including advanced features)..."
make -f posix_Makefile test-posix-extended
;;
"full")
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
echo " This includes: basic, extended, and external test suites"
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
make -f posix_Makefile test-posix-full
;;
*)
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
echo "🚀 Running full POSIX test suite (all tests including external)..."
make -f posix_Makefile test-posix-full
;;
esac
- name: Run external test suites
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
continue-on-error: true
run: |
cd test/fuse_integration
# Point the external suites at the same mount as the Go tests;
# without these the Makefile falls back to a local-disk path.
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Run external tests (may fail on some systems)
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
- name: Generate compliance report
if: always()
run: |
cd test/fuse_integration
make -f posix_Makefile generate-report
- name: Collect system information for debugging
if: failure()
run: |
cd test/fuse_integration
mkdir -p reports/debug
echo "System Information" > reports/debug/system_info.txt
uname -a >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Mount Information" >> reports/debug/system_info.txt
mount >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Disk Space" >> reports/debug/system_info.txt
df -h >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Memory Information" >> reports/debug/system_info.txt
free -h >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "FUSE Information" >> reports/debug/system_info.txt
ls -la /dev/fuse >> reports/debug/system_info.txt
lsmod | grep fuse >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "SeaweedFS Version" >> reports/debug/system_info.txt
weed version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Go Version" >> reports/debug/system_info.txt
go version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Running Processes" >> reports/debug/system_info.txt
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
# Collect SeaweedFS service logs
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
if [ -f /tmp/seaweedfs/master.log ]; then
echo "Master Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
echo "Volume Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
echo "Filer Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
echo "Mount Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
# Copy full logs as separate files
if [ -f /tmp/seaweedfs/master.log ]; then
cp /tmp/seaweedfs/master.log reports/debug/master.log
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
fi
- name: Cleanup SeaweedFS cluster and FUSE mount
if: always()
run: |
stop_process() {
local name=$1
local pidfile=$2
if [ -f "$pidfile" ]; then
local pid=$(cat "$pidfile")
if kill -0 $pid 2>/dev/null; then
echo "Stopping $name process (PID: $pid)..."
kill -TERM $pid || true
sleep 2
kill -KILL $pid 2>/dev/null || true
fi
fi
}
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
MOUNT_POINT="/tmp/seaweedfs/mount"
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "Unmounting FUSE filesystem..."
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
fi
stop_process "mount" "/tmp/seaweedfs/mount.pid"
stop_process "filer" "/tmp/seaweedfs/filer.pid"
stop_process "volume" "/tmp/seaweedfs/volume.pid"
stop_process "master" "/tmp/seaweedfs/master.pid"
pkill -f "weed " || true
fusermount -u $MOUNT_POINT 2>/dev/null || true
umount $MOUNT_POINT 2>/dev/null || true
rm -rf /tmp/seaweedfs || true
echo "Cleanup completed"
- name: Upload test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
- name: Upload test coverage
uses: actions/upload-artifact@v7
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
with:
name: posix-coverage-ubuntu
path: test/fuse_integration/reports/posix_coverage.html
retention-days: 14
- name: Comment PR with results
if: github.event_name == 'pull_request' && always()
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
if (fs.existsSync(reportPath)) {
const report = fs.readFileSync(reportPath, 'utf8');
const comment = `## POSIX Compliance Test Results
**Go Version:** ${{ env.GO_VERSION }}
**FUSE Version:** ${{ matrix.fuse-version }}
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
<details>
<summary>Test Summary</summary>
\`\`\`
${report}
\`\`\`
</details>
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
`;
try {
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: comment
});
} catch (err) {
console.log('Could not post PR comment: ' + err.message);
}
}
posix-compliance-macos:
runs-on: macos-latest
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Install macFUSE
run: |
# Install macFUSE
brew install macfuse
# Note: macFUSE may require system extension approval on macOS
# This step may need manual intervention in some cases
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
- name: Run critical POSIX tests (macOS)
continue-on-error: true # macOS FUSE can be more restrictive
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run basic tests only on macOS due to FUSE limitations
make -f posix_Makefile test-posix-critical
- name: Upload macOS test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-macos
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
security-analysis:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
- name: Install security analysis tools
run: |
echo "Installing staticcheck for security analysis..."
go install honnef.co/go/tools/cmd/staticcheck@latest
- name: Run security analysis on FUSE code
run: |
# Analyze mount and FUSE-related code for security and quality issues
echo "Running staticcheck security analysis..."
# Run staticcheck on FUSE-related code
echo "Analyzing FUSE mount code..."
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
# Convert to JSON format for consistency
echo "{" > gosec-report.json
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
echo " \"issues\": [" >> gosec-report.json
if [ -s staticcheck-report.txt ]; then
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
fi
echo " ]," >> gosec-report.json
echo " \"stats\": {" >> gosec-report.json
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
echo " }" >> gosec-report.json
echo "}" >> gosec-report.json
echo "Security analysis completed"
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
- name: Upload security analysis results
uses: actions/upload-artifact@v7
if: always()
with:
name: security-analysis-results
path: gosec-report.json
retention-days: 30
performance-baseline:
runs-on: ubuntu-latest
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev fio
sudo usermod -a -G fuse $USER
sudo chmod 666 /dev/fuse
- name: Build SeaweedFS
run: make
- name: Run performance baseline tests
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run performance benchmarks
make -f posix_Makefile benchmark-posix
# This job has no FUSE mount, so the fio suite can only skip; keep the
# run green but honest about producing no mounted-FS results
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
- name: Store performance baseline
uses: actions/upload-artifact@v7
with:
name: performance-baseline-results
path: |
test/fuse_integration/reports/posix_benchmark_results.log
test/fuse_integration/reports/fio_*_results.log
retention-days: 90
# Schedule regular compliance checks
scheduled-compliance-check:
runs-on: ubuntu-latest
if: github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev
pip3 install --user nfstest
- name: Build SeaweedFS
run: make
- name: Run comprehensive compliance check
run: |
cd test/fuse_integration
make -f posix_Makefile test-posix-full
make -f posix_Makefile generate-report
- name: Create compliance issue if tests fail
if: failure()
uses: actions/github-script@v7
with:
script: |
const issue = await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
body: `## POSIX Compliance Check Failure
The scheduled POSIX compliance check has failed. This may indicate:
- Regression in FUSE mount functionality
- Changes affecting POSIX compatibility
- Infrastructure issues with the test environment
**Action Required:** Review the test results and investigate any failures.
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
**Date:** ${new Date().toISOString()}
---
This issue was automatically created by the POSIX compliance workflow.
`,
labels: ['bug', 'posix-compliance', 'automated-issue']
});
console.log('Created issue:', issue.data.number);
# Schedule this workflow to run weekly
# Uncomment the following lines to enable scheduled runs:
# schedule:
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
k8s/charts/seaweedfs/Chart.yaml
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.48"
appVersion: "4.49"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.48.2
version: 4.49.0
note/star_history.svg
+1094 -1099
View File
File diff suppressed because it is too large. Load diff
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 55 KiB

After

Width:  |  Height:  |  Size: 55 KiB

seaweed-volume/Cargo.lock
Generated
+40 -15
View File
@@ -86,7 +86,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -510,10 +510,12 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
version = "0.62.5"
version = "0.62.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
]
@@ -563,11 +565,12 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.11.6"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
"aws-smithy-types",
"bytes",
"http 0.2.12",
@@ -579,10 +582,32 @@ dependencies = [
]
[[package]]
name = "aws-smithy-types"
version = "1.4.7"
name = "aws-smithy-runtime-api-macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "aws-smithy-schema"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.0",
]
[[package]]
name = "aws-smithy-types"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
dependencies = [
"base64-simd",
"bytes",
@@ -1325,7 +1350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -2379,7 +2404,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3391,7 +3416,7 @@ dependencies = [
"errno 0.3.14",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3676,7 +3701,7 @@ version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno 0.3.14",
"errno 0.2.8",
"libc",
]
@@ -3743,7 +3768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -3877,7 +3902,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
seaweed-volume/src/storage/store.rs
+86 -4
View File
@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{
};
use crate::storage::needle::needle::{Needle, get_actual_size};
use crate::storage::needle_map::NeedleMapKind;
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
use crate::storage::types::*;
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
@@ -762,8 +762,8 @@ impl Store {
}
/// Configure a volume's replica placement on disk.
/// The volume must already be unmounted. This opens the .dat file directly,
/// modifies the replica_placement byte (offset 1), and writes it back.
/// The volume must already be unmounted. Rewrites the .vif and the .dat
/// superblock so both carry the new placement.
pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> {
let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| {
VolumeError::Io(io::Error::new(
@@ -775,7 +775,7 @@ impl Store {
let mut vif = load_vif_volume_info(&vif_path)?;
vif.replication = rp.to_string();
save_vif_volume_info(&vif_path, &vif)?;
Ok(())
update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp)
}
// ---- Read / Write / Delete ----
@@ -1740,6 +1740,61 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> {
.or_else(|| filename.strip_suffix(".idx"))
}
/// Rewrites the replica placement in a .dat header for a volume that is not
/// mounted; a missing .dat means a remote-only volume, where the .vif carries
/// the placement.
fn update_dat_replica_placement(
dat_path: &str,
vid: VolumeId,
rp: ReplicaPlacement,
) -> Result<(), VolumeError> {
use std::io::{Read, Seek, SeekFrom, Write};
let mut file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(dat_path)
{
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => {
return Err(VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to open {}: {}", vid, dat_path, e),
)))
}
};
let mut header = [0u8; SUPER_BLOCK_SIZE];
file.read_exact(&mut header).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize;
let mut block = header.to_vec();
block.resize(SUPER_BLOCK_SIZE + extra_size, 0);
if extra_size > 0 {
file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
}
let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| {
VolumeError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
super_block.replica_placement = rp;
file.seek(SeekFrom::Start(0))?;
file.write_all(&super_block.to_bytes())?;
file.sync_all()?;
Ok(())
}
fn load_vif_volume_info(path: &str) -> Result<VifVolumeInfo, VolumeError> {
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
@@ -1977,6 +2032,33 @@ mod tests {
assert_eq!(store.total_volume_count(), 1);
}
// Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave
// the .dat header and the .vif both on the new placement.
#[test]
fn test_configure_volume_updates_superblock() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = make_test_store(&[dir]);
store
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
.unwrap();
assert!(store.unmount_volume(VolumeId(1)).unwrap());
let rp = ReplicaPlacement::from_string("010").unwrap();
store.configure_volume(VolumeId(1), rp).unwrap();
let base = volume_file_name(dir, "", VolumeId(1));
let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap();
assert_eq!(vif.replication, "010");
let dat = std::fs::read(format!("{}.dat", base)).unwrap();
let super_block = SuperBlock::from_bytes(&dat).unwrap();
assert_eq!(super_block.replica_placement, rp);
assert!(store.configure_volume(VolumeId(2), rp).is_err());
}
#[test]
fn test_mount_volume_by_id_hint_skips_lone_sidecar() {
// A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a
test/fuse_failover/framework_test.go
+15
View File
@@ -110,6 +110,21 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, c.WaitForVolumeServers(60*time.Second))
// A mounted filesystem and registered volume servers do not yet guarantee
// that the lock ring has settled or the first replicated volume is ready.
// Exercise the write/read path during setup so the append latency checks
// measure failover on a running cluster rather than its cold start.
probeName := ".failover-ready"
probeData := []byte("ready\n")
probePath := filepath.Join(c.MountDir(0), probeName)
require.NoError(t, os.WriteFile(probePath, probeData, 0644),
"readiness write failed\n%s", c.tailLog("mount0"))
for i := 0; i < numMounts; i++ {
_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, os.Remove(probePath))
return c
}
test/fuse_integration/POSIX_COMPLIANCE.md
+645
View File
@@ -0,0 +1,645 @@
# SeaweedFS FUSE POSIX Compliance Testing
## Overview
This comprehensive test suite provides full POSIX compliance testing for SeaweedFS FUSE mounts. It includes basic POSIX operations, extended features, external test suite integration, and performance benchmarking to ensure SeaweedFS meets filesystem standards.
## 🎯 Key Features
### ✅ **Comprehensive Test Coverage**
- **Basic POSIX Operations**: File/directory create, read, write, delete, rename
- **Advanced Features**: Extended attributes, file locking, memory mapping *(see roadmap)*
- **I/O Operations**: Synchronous/asynchronous I/O, direct I/O, vectored I/O *(see roadmap)*
- **Error Handling**: Comprehensive error condition testing
- **Concurrent Operations**: Multi-threaded stress testing
- **External Integration**: pjdfstest, nfstest, FIO integration
> **📋 Implementation Status**: Some advanced features are currently skipped pending platform-specific implementations. See [`POSIX_IMPLEMENTATION_ROADMAP.md`](./POSIX_IMPLEMENTATION_ROADMAP.md) for a detailed implementation plan and current status.
### 📊 **Performance Analysis**
- **Benchmarking**: Built-in performance benchmarks
- **Profiling**: CPU and memory profiling support
- **Coverage Analysis**: Code coverage reporting
- **Stress Testing**: High-load concurrent operation testing
### 🔧 **External Tool Integration**
- **pjdfstest**: Industry-standard POSIX filesystem test suite
- **nfstest_posix**: Network filesystem POSIX API verification
- **FIO**: Flexible I/O performance testing
- **Custom Litmus Tests**: Focused edge case testing
## 📋 Test Categories
### 1. Basic POSIX Compliance (`posix_compliance_test.go`)
#### File Operations
- ✅ File creation with O_CREAT, O_EXCL flags
- ✅ File truncation and size management
- ✅ File deletion and unlinking
- ✅ Atomic rename operations
- ✅ File permission management
#### Directory Operations
- ✅ Directory creation and removal
- ✅ Directory listing and traversal
- ✅ Directory rename operations
- ✅ Non-empty directory handling
#### Symlink Operations
- ✅ Symbolic link creation and resolution
- ✅ Broken symlink handling
- ✅ Symlink target verification
- ✅ Symlink permission handling
#### Permission Tests
- ✅ File permission setting and verification
- ✅ Directory permission enforcement
- ✅ Permission inheritance
- ✅ Access control validation
#### Timestamp Tests
- ✅ Access time (atime) updates
- ✅ Modification time (mtime) tracking
- ✅ Change time (ctime) management
- ✅ Timestamp precision and setting
#### I/O Operations
- ✅ Read/write operations
- ✅ File seeking and positioning
- ✅ Append mode operations
- ✅ Buffer management
#### File Descriptors
- ✅ File descriptor duplication
- ✅ File descriptor flags
- ✅ Close-on-exec handling
- ✅ File descriptor limits
#### Atomic Operations
- ✅ Atomic file operations
- ✅ Rename atomicity
- ✅ Link/unlink atomicity
#### Concurrent Access
- ✅ Multi-reader scenarios
- ✅ Concurrent write handling
- ✅ Reader-writer coordination
- ✅ Race condition prevention
#### Error Handling
- ✅ ENOENT (file not found) handling
- ✅ EACCES (permission denied) handling
- ✅ EBADF (bad file descriptor) handling
- ✅ ENOTEMPTY (directory not empty) handling
- ✅ Error code compliance
### 2. Extended POSIX Features (`posix_extended_test.go`)
#### Extended Attributes
- ✅ Setting extended attributes (setxattr)
- ✅ Getting extended attributes (getxattr)
- ✅ Listing extended attributes (listxattr)
- ✅ Removing extended attributes (removexattr)
- ✅ Extended attribute namespaces
#### File Locking
- ✅ Advisory locking (fcntl)
- ✅ Exclusive locks (F_WRLCK)
- ✅ Shared locks (F_RDLCK)
- ✅ Lock conflict detection
- ✅ Lock release and cleanup
#### Advanced I/O
- ✅ Vectored I/O (readv/writev)
- ✅ Positioned I/O (pread/pwrite)
- ✅ Asynchronous I/O patterns
- ✅ Scatter-gather operations
#### Sparse Files
- ✅ Sparse file creation
- ✅ Hole detection and handling
- ✅ Sparse file efficiency
- ✅ Seek beyond EOF
#### Large Files (>2GB)
- ✅ Large file operations
- ✅ 64-bit offset handling
- ✅ Large file seeking
- ✅ Large file truncation
#### Memory Mapping
- ✅ File memory mapping (mmap)
- ✅ Memory-mapped I/O
- ✅ Memory synchronization (msync)
- ✅ Memory unmapping (munmap)
- ✅ Shared/private mappings
#### Direct I/O
- ✅ Direct I/O operations (O_DIRECT)
- ✅ Buffer alignment requirements
- ✅ Direct I/O performance
- ✅ Bypass buffer cache
#### File Preallocation
- ✅ Space preallocation (fallocate)
- ✅ File hole punching
- ✅ Space reservation
- ✅ Allocation efficiency
#### Zero-Copy Operations
- ✅ Zero-copy file transfer (sendfile)
- ✅ Efficient data movement
- ✅ Cross-filesystem transfers
### 3. External Test Suite Integration (`posix_external_test.go`)
#### pjdfstest Integration
- ✅ Comprehensive POSIX filesystem test suite
- ✅ Industry-standard compliance verification
- ✅ Automated test execution
- ✅ Result analysis and reporting
#### nfstest_posix Integration
- ✅ POSIX API verification
- ✅ Network filesystem compliance
- ✅ API behavior validation
- ✅ Comprehensive coverage
#### Custom Litmus Tests
- ✅ Atomic rename verification
- ✅ Link count accuracy
- ✅ Symlink cycle detection
- ✅ Concurrent operation safety
- ✅ Directory consistency
#### Stress Testing
- ✅ High-frequency operations
- ✅ Concurrent access patterns
- ✅ Resource exhaustion scenarios
- ✅ Error recovery testing
#### Edge Case Testing
- ✅ Empty filenames
- ✅ Very long filenames
- ✅ Special characters
- ✅ Deep directory nesting
- ✅ Path length limits
## 🚀 Quick Start
### Prerequisites
```bash
# Install required tools
sudo apt-get install fuse # Linux
brew install macfuse # macOS
# Install Go 1.21+
go version
# Build SeaweedFS
cd /path/to/seaweedfs
make
```
### Basic Usage
```bash
# Navigate to test directory
cd test/fuse_integration
# Check prerequisites
make -f posix_Makefile check-prereqs
# Run basic POSIX compliance tests
make -f posix_Makefile test-posix-basic
# Run complete test suite
make -f posix_Makefile test-posix-full
# Generate compliance report
make -f posix_Makefile generate-report
```
## 🔧 Advanced Usage
### Test Categories
#### Critical Tests Only
```bash
# Run only critical POSIX compliance tests (faster)
make -f posix_Makefile test-posix-critical
```
#### Extended Feature Tests
```bash
# Test advanced POSIX features
make -f posix_Makefile test-posix-extended
```
#### External Tool Integration
```bash
# Setup and run external test suites
make -f posix_Makefile setup-external-tools
make -f posix_Makefile test-posix-external
```
#### Stress Testing
```bash
# Run stress tests for concurrent operations
make -f posix_Makefile test-posix-stress
```
### Performance Analysis
#### Benchmarking
```bash
# Run performance benchmarks
make -f posix_Makefile benchmark-posix
```
#### Profiling
```bash
# Profile CPU and memory usage
make -f posix_Makefile profile-posix
# View CPU profile
go tool pprof reports/posix.cpu.prof
```
#### Coverage Analysis
```bash
# Generate test coverage report
make -f posix_Makefile coverage-posix
# View: reports/posix_coverage.html
```
#### FIO Performance Testing
```bash
# Run FIO-based I/O performance tests
make -f posix_Makefile test-fio-posix
```
### External Test Suites
#### pjdfstest
```bash
# Setup and run pjdfstest
make -f posix_Makefile setup-pjdfstest
make -f posix_Makefile test-pjdfstest
```
#### nfstest_posix
```bash
# Install and run nfstest_posix
make -f posix_Makefile setup-nfstest
make -f posix_Makefile test-nfstest-posix
```
## 📊 Understanding Results
### Test Status Indicators
- ✅ **PASS**: Test completed successfully
- ❌ **FAIL**: Test failed - indicates non-compliance
- ⚠️ **SKIP**: Test skipped - feature not supported
- 🔄 **RETRY**: Test retried due to transient failure
### Report Formats
#### HTML Report (`reports/posix_compliance_report.html`)
- Interactive web-based report
- Test category breakdown
- Detailed results with navigation
- Visual status indicators
#### Text Summary (`reports/posix_compliance_summary.txt`)
- Concise text-based summary
- Key findings and recommendations
- Command-line friendly format
#### JSON Report (`reports/posix_compliance_report.json`)
- Machine-readable results
- Integration with CI/CD pipelines
- Automated result processing
### Log Files
- `posix_basic_results.log`: Basic POSIX test detailed output
- `posix_extended_results.log`: Extended feature test output
- `posix_external_results.log`: External test suite output
- `posix_benchmark_results.log`: Performance benchmark results
## 🐛 Debugging Failed Tests
### Common Issues
#### Permission Denied Errors
```bash
# Ensure proper FUSE permissions
sudo usermod -a -G fuse $USER
# Re-login or run: newgrp fuse
# Mount with -allowOthers if other users need access (test config MountOptions)
```
#### Extended Attributes Not Supported
```bash
# Check if xattr is supported by underlying filesystem
getfattr --version
# May need to skip xattr tests on certain filesystems
```
#### File Locking Issues
```bash
# Verify file locking support
# Some network filesystems may not support full locking
```
#### Memory Issues
```bash
# Increase available memory for tests
# Large file tests may require significant RAM
```
### Debug Mode
```bash
# Test targets already run go test -v and keep logs under reports/
make -f posix_Makefile test-posix-basic
# Run against an existing external mount instead of a fresh cluster
make -f posix_Makefile test-posix-basic TEST_MOUNT_POINT=/path/to/mount
# Extend the 45-minute per-suite timeout
make -f posix_Makefile test-posix-full TEST_TIMEOUT=90m
```
### Manual Test Execution
```bash
# Run specific test
cd test/fuse_integration
go test -v -run 'TestPOSIXCompliance/FileOperations' .
# Run with custom timeout
go test -v -timeout 30m -run TestPOSIXExtended .
```
## 🔄 CI/CD Integration
### GitHub Actions
```yaml
name: POSIX Compliance Tests
on: [push, pull_request]
jobs:
posix-compliance:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Install FUSE
run: sudo apt-get update && sudo apt-get install -y fuse
- name: Build SeaweedFS
run: make
- name: Run POSIX Compliance Tests
run: |
cd test/fuse_integration
make -f posix_Makefile ci-posix-tests
- name: Upload Test Results
uses: actions/upload-artifact@v4
if: always()
with:
name: posix-test-results
path: test/fuse_integration/reports/
```
### Jenkins Pipeline
```groovy
pipeline {
agent any
stages {
stage('Setup') {
steps {
sh 'make'
}
}
stage('POSIX Compliance') {
steps {
dir('test/fuse_integration') {
sh 'make -f posix_Makefile ci-posix-tests'
}
}
post {
always {
archiveArtifacts 'test/fuse_integration/reports/**'
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: 'test/fuse_integration/reports',
reportFiles: 'posix_compliance_report.html',
reportName: 'POSIX Compliance Report'
])
}
}
}
}
}
```
## 🎯 POSIX Compliance Checklist
### Critical POSIX Features
- [ ] File creation, reading, writing, deletion
- [ ] Directory operations (create, remove, list)
- [ ] File permissions and ownership
- [ ] Symbolic links
- [ ] Hard links
- [ ] File timestamps
- [ ] Error handling compliance
- [ ] Atomic operations
### Advanced POSIX Features
- [ ] Extended attributes
- [ ] File locking (advisory)
- [ ] Memory-mapped I/O
- [ ] Vectored I/O (readv/writev)
- [ ] Positioned I/O (pread/pwrite)
- [ ] Direct I/O
- [ ] File preallocation
- [ ] Sparse files
### Performance Requirements
- [ ] Reasonable performance under load
- [ ] Concurrent access handling
- [ ] Memory usage optimization
- [ ] CPU efficiency
- [ ] I/O throughput
### Reliability Requirements
- [ ] Data consistency
- [ ] Error recovery
- [ ] Race condition prevention
- [ ] Resource cleanup
- [ ] Crash recovery
## 📈 Performance Benchmarks
### Expected Performance Characteristics
| Operation | Expected Range | Notes |
|-----------|----------------|-------|
| File Create | 1000-10000 ops/sec | Depends on chunk size |
| File Read | 100-1000 MB/sec | Network limited |
| File Write | 50-500 MB/sec | Depends on replication |
| Directory Ops | 500-5000 ops/sec | Metadata operations |
| Concurrent Ops | Scales with cores | Up to hardware limits |
### Benchmarking Commands
```bash
# Basic I/O performance
make -f posix_Makefile benchmark-posix
# FIO comprehensive testing
make -f posix_Makefile test-fio-posix
# Custom benchmark with specific parameters
go test -v -bench=. -benchtime=30s posix_compliance_test.go
```
## 🔍 Troubleshooting
### Common Test Failures
#### 1. Mount Permission Issues
```bash
# Error: Permission denied mounting
# Solution: Check FUSE permissions — add the user to the fuse group and
# re-login rather than widening permissions on the device node
sudo usermod -a -G fuse $USER
```
#### 2. Extended Attribute Tests Fail
```bash
# Error: Operation not supported (ENOTSUP)
# Solution: Check filesystem xattr support
mount | grep $(df . | tail -1 | awk '{print $1}')
# May need to remount with xattr support
```
#### 3. File Locking Tests Fail
```bash
# Error: Function not implemented (ENOSYS)
# Solution: Some network filesystems don't support locking
# This may be expected behavior
```
#### 4. Large File Tests Fail
```bash
# Error: No space left on device
# Solution: Ensure sufficient disk space
df -h /tmp # Check temp directory space
# May need to clean up or use different temp directory
```
#### 5. Performance Tests Timeout
```bash
# Error: Test timeout exceeded
# Solution: Increase timeout or reduce test scope
go test -timeout 60m -run TestPOSIXCompliance
```
### Debug Information Collection
```bash
# Collect system information
uname -a > debug_info.txt
mount >> debug_info.txt
df -h >> debug_info.txt
free -h >> debug_info.txt
# Collect SeaweedFS information
./weed version >> debug_info.txt
ps aux | grep weed >> debug_info.txt
# Collect test logs
cp reports/*.log debug_logs/
```
## 📚 Additional Resources
### POSIX Standards
- [POSIX.1-2017 (IEEE Std 1003.1-2017)](https://pubs.opengroup.org/onlinepubs/9699919799/)
- [Single UNIX Specification](https://www.opengroup.org/openbrand/register/)
### External Test Suites
- [pjdfstest](https://github.com/pjd/pjdfstest) - Comprehensive POSIX filesystem tests
- [nfstest](https://nfstest.readthedocs.io/) - Network filesystem testing
- [FIO](https://fio.readthedocs.io/) - Flexible I/O tester
### SeaweedFS Documentation
- [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
- [FUSE Mount Documentation](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount)
- [Performance Tuning Guide](https://github.com/seaweedfs/seaweedfs/wiki/Optimization)
## 🤝 Contributing
### Adding New Tests
1. **Basic Tests**: Add to `posix_compliance_test.go`
2. **Extended Tests**: Add to `posix_extended_test.go`
3. **External Integration**: Add to `posix_external_test.go`
### Test Guidelines
```go
func (s *POSIXTestSuite) TestNewFeature(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("SubTestName", func(t *testing.T) {
// Test setup
testFile := filepath.Join(mountPoint, "test.txt")
// Test execution
err := someOperation(testFile)
// Assertions
require.NoError(t, err)
// Cleanup (if needed)
defer os.Remove(testFile)
})
}
```
### Submitting Improvements
1. Fork the repository
2. Create a feature branch
3. Add comprehensive tests
4. Ensure all existing tests pass
5. Submit a pull request with detailed description
## 📞 Support
For questions, issues, or contributions:
- **GitHub Issues**: [SeaweedFS Issues](https://github.com/seaweedfs/seaweedfs/issues)
- **Discussions**: [SeaweedFS Discussions](https://github.com/seaweedfs/seaweedfs/discussions)
- **Documentation**: [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
---
*This comprehensive POSIX compliance test suite ensures SeaweedFS FUSE mounts meet industry standards for filesystem behavior, performance, and reliability.*
test/fuse_integration/POSIX_IMPLEMENTATION_ROADMAP.md
+272
View File
@@ -0,0 +1,272 @@
# POSIX Compliance Implementation Roadmap
This document tracks the implementation status of POSIX features in the SeaweedFS FUSE mount compliance test suite and provides a roadmap for completing comprehensive POSIX compliance testing.
## Overview
The platform-specific tests listed below are implemented; this document records the implementation plan that produced them, plus the one feature still pending.
## Current Status
### ✅ Implemented Features
- Basic file operations (create, read, write, delete)
- Directory operations (mkdir, rmdir, rename)
- File permissions and ownership
- Symbolic and hard links
- Basic I/O operations (seek, append, positioned I/O)
- File descriptors and atomic operations
- Concurrent access patterns
- Error handling compliance
- Timestamp operations
### 🚧 Partially Implemented
- Cross-platform compatibility (basic implementation with platform-specific access time handling)
- File sealing (still skipped; Linux-specific, pending)
### ✅ Implemented Platform-Specific Features
#### 1. Extended Attributes (xattr)
**Priority: High**
**Platforms: Linux, macOS, FreeBSD**
**Status:** Implemented
- `TestExtendedAttributes/SetExtendedAttribute`
- `TestExtendedAttributes/ListExtendedAttributes`
- `TestExtendedAttributes/RemoveExtendedAttribute`
**Implementation Plan:**
```go
// Linux implementation
//go:build linux
func setXattr(path, name string, value []byte) error {
return syscall.Setxattr(path, name, value, 0)
}
// macOS implementation
//go:build darwin
func setXattr(path, name string, value []byte) error {
return syscall.Setxattr(path, name, value, 0, 0)
}
```
**Required Files:**
- `xattr_linux.go` - Linux syscall implementations
- `xattr_darwin.go` - macOS syscall implementations
- `xattr_freebsd.go` - FreeBSD syscall implementations
- `xattr_unsupported.go` - Fallback for unsupported platforms
#### 2. Memory Mapping (mmap)
**Priority: High**
**Platforms: All POSIX systems**
**Status:** Implemented
- `TestMemoryMapping/MemoryMappedRead`
- `TestMemoryMapping/MemoryMappedWrite`
**Implementation Plan:**
```go
func testMmap(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Platform-specific mmap implementation
data, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
require.NoError(t, err)
defer syscall.Munmap(data)
}
```
**Required Files:**
- `mmap_unix.go` - Unix-like systems implementation
- `mmap_windows.go` - Windows implementation (if needed)
#### 3. Vectored I/O (readv/writev)
**Priority: Medium**
**Platforms: All POSIX systems**
**Status:** Implemented
- `TestAdvancedIO/VectoredIO`
**Implementation Plan:**
```go
func testVectoredIO(t *testing.T, filePath string) {
// Use syscall.Syscall for readv/writev
// Linux: SYS_READV, SYS_WRITEV
// Other platforms: platform-specific syscall numbers
}
```
#### 4. Direct I/O
**Priority: Medium**
**Platforms: Linux, some Unix variants**
**Status:** Implemented
- `TestDirectIO/DirectIO`
**Implementation Plan:**
```go
//go:build linux
func testDirectIO(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR|syscall.O_DIRECT, 0)
// Test with aligned buffers
}
```
#### 5. File Preallocation (fallocate)
**Priority: Medium**
**Platforms: Linux, some Unix variants**
**Status:** Implemented
- `TestFilePreallocation/Fallocate`
**Implementation Plan:**
```go
//go:build linux
func testFallocate(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
err = syscall.Syscall6(syscall.SYS_FALLOCATE, uintptr(fd), 0, 0, uintptr(size), 0, 0)
}
```
#### 6. Zero-Copy Transfer (sendfile)
**Priority: Low**
**Platforms: Linux, FreeBSD, some Unix variants**
**Status:** Implemented
- `TestZeroCopyTransfer/Sendfile`
**Implementation Plan:**
```go
//go:build linux
func testSendfile(t *testing.T, srcPath, dstPath string) {
// Use syscall.Syscall for sendfile
// Platform-specific implementations
}
```
#### 7. File Sealing (Linux-specific)
**Priority: Low**
**Platforms: Linux only**
**Current Status:** File sealing test is still skipped
- `TestFileSealing`
**Implementation Plan:**
```go
//go:build linux
func testFileSealing(t *testing.T) {
// Use fcntl with F_ADD_SEALS, F_GET_SEALS
// Test various seal types: F_SEAL_WRITE, F_SEAL_SHRINK, etc.
}
```
## Implementation Strategy
### Phase 1: Core Features (High Priority)
1. **Extended Attributes** - Essential for many applications
2. **Memory Mapping** - Critical for performance-sensitive applications
### Phase 2: Advanced I/O (Medium Priority)
3. **Vectored I/O** - Important for efficient bulk operations
4. **Direct I/O** - Needed for database and high-performance applications
5. **File Preallocation** - Important for preventing fragmentation
### Phase 3: Specialized Features (Low Priority)
6. **Zero-Copy Transfer** - Performance optimization feature
7. **File Sealing** - Security feature, Linux-specific
## Platform Support Matrix
| Feature | Linux | macOS | FreeBSD | Windows | Notes |
|---------|-------|-------|---------|---------|-------|
| Extended Attributes | ✅ | ✅ | ✅ | ❌ | Different syscall signatures |
| Memory Mapping | ✅ | ✅ | ✅ | ✅ | Standard POSIX |
| Vectored I/O | ✅ | ✅ | ✅ | ❌ | readv/writev syscalls |
| Direct I/O | ✅ | ❌ | ✅ | ❌ | O_DIRECT flag |
| fallocate | ✅ | ❌ | ❌ | ❌ | Linux-specific |
| sendfile | ✅ | ❌ | ✅ | ❌ | Platform-specific |
| File Sealing | ✅ | ❌ | ❌ | ❌ | Linux-only |
## Development Guidelines
### 1. Platform-Specific Implementation Pattern
```go
// feature_linux.go
//go:build linux
package fuse
func platformSpecificFunction() { /* Linux implementation */ }
// feature_darwin.go
//go:build darwin
package fuse
func platformSpecificFunction() { /* macOS implementation */ }
// feature_unsupported.go
//go:build !linux && !darwin
package fuse
func platformSpecificFunction() { /* Skip or error */ }
```
### 2. Test Structure Pattern
```go
func (s *POSIXExtendedTestSuite) TestFeature(t *testing.T) {
if !isFeatureSupported() {
t.Skip("Feature not supported on this platform")
return
}
t.Run("FeatureTest", func(t *testing.T) {
// Actual test implementation
})
}
```
### 3. Error Handling
- Use platform-specific error checking
- Provide meaningful error messages
- Gracefully handle unsupported features
## Testing Strategy
### 1. Continuous Integration
- Run tests on multiple platforms (Linux, macOS)
- Use build tags to enable/disable platform-specific tests
- Ensure graceful degradation on unsupported platforms
### 2. Feature Detection
- Implement runtime feature detection where possible
- Skip tests gracefully when features are unavailable
- Log clear messages about skipped functionality
### 3. Documentation
- Document platform-specific behavior
- Provide examples for each implemented feature
- Maintain compatibility matrices
## Contributing
When implementing these features:
1. **Start with high-priority items** (Extended Attributes, Memory Mapping)
2. **Follow the platform-specific pattern** outlined above
3. **Add comprehensive tests** for each feature
4. **Update this roadmap** as features are implemented
5. **Document any platform-specific quirks** or limitations
## Future Enhancements
Beyond the current roadmap, consider:
- **File locking** (flock, fcntl locks)
- **Asynchronous I/O** (aio_read, aio_write)
- **File change notifications** (inotify, kqueue)
- **POSIX ACLs** (Access Control Lists)
- **Sparse file operations**
- **File hole punching**
## References
- [POSIX.1-2017 Standard](https://pubs.opengroup.org/onlinepubs/9699919799/)
- [Linux Programmer's Manual](https://man7.org/linux/man-pages/)
- [FreeBSD System Calls](https://www.freebsd.org/cgi/man.cgi)
- [macOS System Calls](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/)
test/fuse_integration/atime_linux.go
+12
View File
@@ -0,0 +1,12 @@
//go:build linux
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for Linux systems
func getAtimeNano(stat *syscall.Stat_t) int64 {
return stat.Atim.Sec*1e9 + stat.Atim.Nsec
}
test/fuse_integration/atime_nonlinux.go
+12
View File
@@ -0,0 +1,12 @@
//go:build !linux && !windows
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for non-Linux systems (macOS, BSD, etc.)
func getAtimeNano(stat *syscall.Stat_t) int64 {
return stat.Atimespec.Sec*1e9 + stat.Atimespec.Nsec
}
test/fuse_integration/atime_windows.go
+12
View File
@@ -0,0 +1,12 @@
//go:build windows
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for Windows systems
func getAtimeNano(stat *syscall.Win32FileAttributeData) int64 {
return stat.LastAccessTime.Nanoseconds()
}
test/fuse_integration/directio_darwin.go
+39
View File
@@ -0,0 +1,39 @@
//go:build darwin
package fuse
import (
"syscall"
)
// Direct I/O support for macOS
const (
// macOS doesn't have O_DIRECT, but we can use fcntl with F_NOCACHE
F_NOCACHE = 48
)
func openDirectIO(path string, flags int, mode uint32) (int, error) {
// Open file normally first
fd, err := syscall.Open(path, flags, mode)
if err != nil {
return -1, err
}
// Set F_NOCACHE to bypass buffer cache (similar to O_DIRECT)
_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
uintptr(fd),
F_NOCACHE,
1) // enable
if errno != 0 {
syscall.Close(fd)
return -1, errno
}
return fd, nil
}
func isDirectIOSupported() bool {
return true
}
test/fuse_integration/directio_linux.go
+23
View File
@@ -0,0 +1,23 @@
//go:build linux
package fuse
import (
"syscall"
"golang.org/x/sys/unix"
)
// Direct I/O support for Linux
const (
O_DIRECT = unix.O_DIRECT
)
func openDirectIO(path string, flags int, mode uint32) (int, error) {
return syscall.Open(path, flags|O_DIRECT, mode)
}
func isDirectIOSupported() bool {
return true
}
test/fuse_integration/directio_unsupported.go
+29
View File
@@ -0,0 +1,29 @@
//go:build !linux && !darwin
package fuse
import (
"errors"
"syscall"
)
// Direct I/O support for unsupported platforms
var ErrDirectIONotSupported = errors.New("direct I/O not supported on this platform")
const (
O_DIRECT = 0x4000 // Dummy flag for compatibility
)
func openDirectIO(path string, flags int, mode uint32) (int, error) {
// Fall back to regular open
fd, err := syscall.Open(path, flags, mode)
if err != nil {
return -1, err
}
return int(fd), nil
}
func isDirectIOSupported() bool {
return false
}
test/fuse_integration/fallocate_darwin.go
+79
View File
@@ -0,0 +1,79 @@
//go:build darwin
package fuse
import (
"syscall"
"unsafe"
)
// File allocation support for macOS using fcntl
const (
// macOS doesn't have fallocate, but we can use fcntl with F_PREALLOCATE
F_ALLOCATECONTIG = 0x02
F_ALLOCATEALL = 0x04
// Dummy flags for compatibility
FALLOC_FL_KEEP_SIZE = 0x01
FALLOC_FL_PUNCH_HOLE = 0x02
FALLOC_FL_NO_HIDE_STALE = 0x04
FALLOC_FL_COLLAPSE_RANGE = 0x08
FALLOC_FL_ZERO_RANGE = 0x10
FALLOC_FL_INSERT_RANGE = 0x20
FALLOC_FL_UNSHARE_RANGE = 0x40
)
// fstore_t structure for F_PREALLOCATE
type fstore struct {
flags uint32
posmode int16
offset int64
length int64
bytesalloc int64
}
func fallocateFile(fd int, mode int, offset int64, length int64) error {
// Check for unsupported modes on macOS
unsupportedModes := FALLOC_FL_PUNCH_HOLE | FALLOC_FL_NO_HIDE_STALE | FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE
if mode&unsupportedModes != 0 {
return syscall.ENOTSUP // Operation not supported
}
// On macOS, we use fcntl with F_PREALLOCATE
store := fstore{
flags: F_ALLOCATECONTIG,
posmode: syscall.F_PEOFPOSMODE, // Allocate from EOF
offset: 0,
length: length,
}
_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
uintptr(fd),
syscall.F_PREALLOCATE,
uintptr(unsafe.Pointer(&store)))
if errno != 0 {
// If contiguous allocation fails, try non-contiguous
store.flags = F_ALLOCATEALL
_, _, errno = syscall.Syscall(syscall.SYS_FCNTL,
uintptr(fd),
syscall.F_PREALLOCATE,
uintptr(unsafe.Pointer(&store)))
if errno != 0 {
return errno
}
}
// Set file size if not keeping size
if mode&FALLOC_FL_KEEP_SIZE == 0 {
return syscall.Ftruncate(fd, offset+length)
}
return nil
}
func isFallocateSupported() bool {
return true
}
test/fuse_integration/fallocate_linux.go
+38
View File
@@ -0,0 +1,38 @@
//go:build linux
package fuse
import (
"syscall"
)
// File allocation support for Linux
const (
// Fallocate flags
FALLOC_FL_KEEP_SIZE = 0x01
FALLOC_FL_PUNCH_HOLE = 0x02
FALLOC_FL_NO_HIDE_STALE = 0x04
FALLOC_FL_COLLAPSE_RANGE = 0x08
FALLOC_FL_ZERO_RANGE = 0x10
FALLOC_FL_INSERT_RANGE = 0x20
FALLOC_FL_UNSHARE_RANGE = 0x40
)
func fallocateFile(fd int, mode int, offset int64, length int64) error {
_, _, errno := syscall.Syscall6(syscall.SYS_FALLOCATE,
uintptr(fd),
uintptr(mode),
uintptr(offset),
uintptr(length),
0, 0)
if errno != 0 {
return errno
}
return nil
}
func isFallocateSupported() bool {
return true
}
test/fuse_integration/fallocate_unsupported.go
+30
View File
@@ -0,0 +1,30 @@
//go:build !linux && !darwin
package fuse
import (
"errors"
)
// File allocation support for unsupported platforms
var ErrFallocateNotSupported = errors.New("fallocate not supported on this platform")
const (
// Dummy flags for compatibility
FALLOC_FL_KEEP_SIZE = 0x01
FALLOC_FL_PUNCH_HOLE = 0x02
FALLOC_FL_NO_HIDE_STALE = 0x04
FALLOC_FL_COLLAPSE_RANGE = 0x08
FALLOC_FL_ZERO_RANGE = 0x10
FALLOC_FL_INSERT_RANGE = 0x20
FALLOC_FL_UNSHARE_RANGE = 0x40
)
func fallocateFile(fd int, mode int, offset int64, length int64) error {
return ErrFallocateNotSupported
}
func isFallocateSupported() bool {
return false
}
test/fuse_integration/framework.go
+496
View File
@@ -0,0 +1,496 @@
package fuse
import (
"fmt"
"io/fs"
"net"
"os"
"os/exec"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// FuseTestFramework provides utilities for FUSE integration testing
type FuseTestFramework struct {
t *testing.T
tempDir string
mountPoint string
dataDir string
masterProcess *os.Process
volumeProcess *os.Process
filerProcess *os.Process
mountProcess *os.Process
masterAddr string
volumeAddr string
filerAddr string
weedBinary string
isSetup bool
config *TestConfig
externalMount bool
}
// TestConfig holds configuration for FUSE tests
type TestConfig struct {
Collection string
Replication string
ChunkSizeMB int
CacheSizeMB int
NumVolumes int
EnableDebug bool
MountOptions []string
SkipCleanup bool // for debugging failed tests
}
// DefaultTestConfig returns a default configuration for FUSE tests
func DefaultTestConfig() *TestConfig {
return &TestConfig{
Collection: "",
Replication: "000",
ChunkSizeMB: 4,
CacheSizeMB: 100,
NumVolumes: 3,
EnableDebug: false,
MountOptions: []string{},
SkipCleanup: false,
}
}
// NewFuseTestFramework creates a new FUSE testing framework
func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
if config == nil {
config = DefaultTestConfig()
}
tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
require.NoError(t, err)
return &FuseTestFramework{
t: t,
tempDir: tempDir,
mountPoint: filepath.Join(tempDir, "mount"),
dataDir: filepath.Join(tempDir, "data"),
masterAddr: "127.0.0.1:19333",
volumeAddr: "127.0.0.1:18080",
filerAddr: "127.0.0.1:18888",
weedBinary: findWeedBinary(),
isSetup: false,
config: config,
}
}
// Setup starts SeaweedFS cluster and mounts FUSE filesystem
func (f *FuseTestFramework) Setup(config *TestConfig) error {
if f.isSetup {
return fmt.Errorf("framework already setup")
}
if config != nil {
f.config = config
}
config = f.config
// Check if we should skip cluster setup and use existing mount
if os.Getenv("TEST_SKIP_CLUSTER_SETUP") == "true" {
// Use existing mount point from environment
if existingMount := os.Getenv("TEST_MOUNT_POINT"); existingMount != "" {
f.externalMount = true
// Verify mount point is accessible
if _, err := os.Stat(existingMount); err != nil {
return fmt.Errorf("existing mount point not accessible: %v", err)
}
// A plain directory passes every check below; refuse to run the suite
// against the local disk
if !mountedAt(existingMount) {
return fmt.Errorf("mount point %s is not a mounted filesystem", existingMount)
}
// So can a non-FUSE mount like tmpfs: a device boundary alone does
// not make the results FUSE compliance
if !fuseFilesystem(existingMount) {
return fmt.Errorf("mount point %s is not a FUSE filesystem", existingMount)
}
// Run inside a private work dir so test files do not pile up in
// the shared mount root and cleanup can remove only what it made
workDir, err := os.MkdirTemp(existingMount, ".fuse_test_")
if err != nil {
return fmt.Errorf("creating work dir under existing mount: %v", err)
}
f.mountPoint = workDir
f.t.Logf("Using existing mount point: %s (work dir %s)", existingMount, f.mountPoint)
// Test basic functionality
testFile := filepath.Join(f.mountPoint, ".framework_test")
if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil {
return fmt.Errorf("mount point not writable: %v", err)
}
if err := os.Remove(testFile); err != nil {
f.t.Logf("Warning: failed to cleanup test file: %v", err)
}
// Verify this is actually a SeaweedFS mount by checking for SeaweedFS-specific behavior
// Create a test file and verify it appears in the filer
verifyFile := filepath.Join(f.mountPoint, ".seaweedfs_mount_verification")
if err := os.WriteFile(verifyFile, []byte("SeaweedFS mount verification"), 0644); err != nil {
return fmt.Errorf("mount point verification failed - cannot write: %v", err)
}
// Read it back to ensure it's working
if data, err := os.ReadFile(verifyFile); err != nil {
return fmt.Errorf("mount point verification failed - cannot read: %v", err)
} else if string(data) != "SeaweedFS mount verification" {
return fmt.Errorf("mount point verification failed - data mismatch")
}
// Clean up verification file
if err := os.Remove(verifyFile); err != nil {
f.t.Logf("Warning: failed to cleanup verification file: %v", err)
}
f.t.Logf("✅ SeaweedFS mount point verified and working: %s", f.mountPoint)
f.isSetup = true
return nil
}
}
// Create directories for full cluster setup
dirs := []string{f.mountPoint, f.dataDir}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %v", dir, err)
}
}
// Start master
if err := f.startMaster(config); err != nil {
return fmt.Errorf("failed to start master: %v", err)
}
// Wait for master to be ready
if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil {
return fmt.Errorf("master not ready: %v", err)
}
// Start volume servers
if err := f.startVolumeServers(config); err != nil {
return fmt.Errorf("failed to start volume servers: %v", err)
}
// Wait for volume server to be ready
if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil {
return fmt.Errorf("volume server not ready: %v", err)
}
// Start filer
if err := f.startFiler(config); err != nil {
return fmt.Errorf("failed to start filer: %v", err)
}
// Wait for filer to be ready
if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
return fmt.Errorf("filer not ready: %v", err)
}
// Skip rather than fail on a host with no FUSE device to mount through
if !fuseAvailable() {
f.t.Skip("no FUSE device node (/dev/fuse, /dev/macfuse0, /dev/osxfuse0)")
}
// Mount FUSE filesystem
if err := f.mountFuse(config); err != nil {
return fmt.Errorf("failed to mount FUSE: %v", err)
}
// Wait for mount to be ready
if err := f.waitForMount(30 * time.Second); err != nil {
return fmt.Errorf("FUSE mount not ready: %v", err)
}
f.isSetup = true
return nil
}
// Cleanup stops all processes and removes temporary files
func (f *FuseTestFramework) Cleanup() {
// An external mount needs no teardown; only this run's work dir is ours.
// A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster
// still owns its processes and must not skip stopping them.
if f.externalMount {
if !f.config.SkipCleanup {
os.RemoveAll(f.mountPoint)
os.RemoveAll(f.tempDir)
}
return
}
if f.mountProcess != nil {
f.unmountFuse()
}
// Stop processes in reverse order
processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
for _, proc := range processes {
if proc != nil {
proc.Signal(syscall.SIGTERM)
proc.Wait()
}
}
// Remove temp directory
if !f.config.SkipCleanup {
os.RemoveAll(f.tempDir)
}
}
// GetMountPoint returns the FUSE mount point path
func (f *FuseTestFramework) GetMountPoint() string {
return f.mountPoint
}
// GetFilerAddr returns the filer address
func (f *FuseTestFramework) GetFilerAddr() string {
return f.filerAddr
}
// startMaster starts the SeaweedFS master server
func (f *FuseTestFramework) startMaster(config *TestConfig) error {
var args []string
if config.EnableDebug {
// -v is a global flag; before the subcommand or flag parsing dies
args = append(args, "-v=4")
}
args = append(args,
"master",
"-ip=127.0.0.1",
"-port=19333",
"-mdir=" + filepath.Join(f.dataDir, "master"),
"-raftBootstrap",
)
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if err := cmd.Start(); err != nil {
return err
}
f.masterProcess = cmd.Process
return nil
}
// startVolumeServers starts SeaweedFS volume servers
func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error {
var args []string
if config.EnableDebug {
args = append(args, "-v=4")
}
args = append(args,
"volume",
"-mserver=" + f.masterAddr,
"-ip=127.0.0.1",
"-port=18080",
"-dir=" + filepath.Join(f.dataDir, "volume"),
fmt.Sprintf("-max=%d", config.NumVolumes),
)
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if err := cmd.Start(); err != nil {
return err
}
f.volumeProcess = cmd.Process
return nil
}
// startFiler starts the SeaweedFS filer server
func (f *FuseTestFramework) startFiler(config *TestConfig) error {
var args []string
if config.EnableDebug {
args = append(args, "-v=4")
}
args = append(args,
"filer",
"-master=" + f.masterAddr,
"-ip=127.0.0.1",
"-port=18888",
)
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if err := cmd.Start(); err != nil {
return err
}
f.filerProcess = cmd.Process
return nil
}
// mountFuse mounts the SeaweedFS FUSE filesystem
func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
var args []string
if config.EnableDebug {
args = append(args, "-v=4")
}
args = append(args,
"mount",
"-filer=" + f.filerAddr,
"-dir=" + f.mountPoint,
"-filer.path=/",
"-dirAutoCreate",
)
if config.Collection != "" {
args = append(args, "-collection="+config.Collection)
}
if config.Replication != "" {
args = append(args, "-replication="+config.Replication)
}
if config.ChunkSizeMB > 0 {
args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
}
if config.CacheSizeMB > 0 {
args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB))
}
args = append(args, config.MountOptions...)
cmd := exec.Command(f.weedBinary, args...)
cmd.Dir = f.tempDir
if err := cmd.Start(); err != nil {
return err
}
f.mountProcess = cmd.Process
return nil
}
// unmountFuse unmounts the FUSE filesystem
func (f *FuseTestFramework) unmountFuse() error {
if f.mountProcess != nil {
f.mountProcess.Signal(syscall.SIGTERM)
f.mountProcess.Wait()
f.mountProcess = nil
}
// Also try system unmount as backup
exec.Command("umount", f.mountPoint).Run()
return nil
}
// waitForService waits for a service to be available
func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
if err == nil {
conn.Close()
return nil
}
time.Sleep(100 * time.Millisecond)
}
return fmt.Errorf("service at %s not ready within timeout", addr)
}
// waitForMount waits for the FUSE mount to be ready
func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
// The mount point is created before weed mount starts, so Stat/ReadDir
// alone succeed on the plain directory. The device id only differs from
// its parent's once a filesystem is actually mounted on it.
parentDev, err := deviceID(filepath.Dir(f.mountPoint))
if err != nil {
return fmt.Errorf("stat mount parent: %w", err)
}
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if dev, err := deviceID(f.mountPoint); err == nil && dev != parentDev {
if _, err := os.ReadDir(f.mountPoint); err == nil {
return nil
}
}
time.Sleep(100 * time.Millisecond)
}
return fmt.Errorf("mount point not ready within timeout")
}
// fuseAvailable reports whether the host has a FUSE device node to mount
// through: /dev/fuse on Linux, /dev/macfuse* or /dev/osxfuse* on macOS.
func fuseAvailable() bool {
for _, dev := range []string{"/dev/fuse", "/dev/macfuse0", "/dev/osxfuse0"} {
if _, err := os.Stat(dev); err == nil {
return true
}
}
return false
}
// findWeedBinary locates the weed binary. Candidates are resolved to
// absolute paths because the started processes run with cmd.Dir set to the
// framework's temp dir, where a relative path would not resolve.
func findWeedBinary() string {
if p, err := exec.LookPath("weed"); err == nil {
return p
}
candidates := []string{
"../../weed/weed",
"./weed",
"../weed",
}
for _, candidate := range candidates {
if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0111 != 0 {
if abs, err := filepath.Abs(candidate); err == nil {
return abs
}
}
}
// Default fallback
return "weed"
}
// Helper functions for test assertions
// AssertFileExists checks if a file exists in the mount point
func (f *FuseTestFramework) AssertFileExists(relativePath string) {
fullPath := filepath.Join(f.mountPoint, relativePath)
_, err := os.Stat(fullPath)
require.NoError(f.t, err, "file should exist: %s", relativePath)
}
// AssertFileNotExists checks if a file does not exist in the mount point
func (f *FuseTestFramework) AssertFileNotExists(relativePath string) {
fullPath := filepath.Join(f.mountPoint, relativePath)
_, err := os.Stat(fullPath)
require.True(f.t, os.IsNotExist(err), "file should not exist: %s", relativePath)
}
// AssertFileContent checks if a file has expected content
func (f *FuseTestFramework) AssertFileContent(relativePath string, expectedContent []byte) {
fullPath := filepath.Join(f.mountPoint, relativePath)
actualContent, err := os.ReadFile(fullPath)
require.NoError(f.t, err, "failed to read file: %s", relativePath)
require.Equal(f.t, expectedContent, actualContent, "file content mismatch: %s", relativePath)
}
// AssertFileMode checks if a file has expected permissions
func (f *FuseTestFramework) AssertFileMode(relativePath string, expectedMode fs.FileMode) {
fullPath := filepath.Join(f.mountPoint, relativePath)
info, err := os.Stat(fullPath)
require.NoError(f.t, err, "failed to stat file: %s", relativePath)
require.Equal(f.t, expectedMode, info.Mode(), "file mode mismatch: %s", relativePath)
}
// CreateTestFile creates a test file with specified content
func (f *FuseTestFramework) CreateTestFile(relativePath string, content []byte) {
fullPath := filepath.Join(f.mountPoint, relativePath)
dir := filepath.Dir(fullPath)
require.NoError(f.t, os.MkdirAll(dir, 0755), "failed to create directory: %s", dir)
require.NoError(f.t, os.WriteFile(fullPath, content, 0644), "failed to create file: %s", relativePath)
}
// CreateTestDir creates a test directory
func (f *FuseTestFramework) CreateTestDir(relativePath string) {
fullPath := filepath.Join(f.mountPoint, relativePath)
require.NoError(f.t, os.MkdirAll(fullPath, 0755), "failed to create directory: %s", relativePath)
}
test/fuse_integration/framework_test.go
+7 -1
View File
@@ -30,6 +30,7 @@ type FuseTestFramework struct {
filerPort int
weedBinary string
isSetup bool
config *TestConfig
}
// TestConfig holds configuration for FUSE tests
@@ -83,6 +84,7 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort),
weedBinary: findWeedBinary(),
isSetup: false,
config: config,
}
}
@@ -132,6 +134,10 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
if f.isSetup {
return fmt.Errorf("framework already setup")
}
if config != nil {
f.config = config
}
config = f.config
dirs := []string{f.mountPoint, f.logDir, f.dataDir}
for _, dir := range dirs {
@@ -182,7 +188,7 @@ func (f *FuseTestFramework) Cleanup() {
f.copyLogsForCI()
if !DefaultTestConfig().SkipCleanup {
if !f.config.SkipCleanup {
os.RemoveAll(f.tempDir)
}
}
test/fuse_integration/fuse_mount_bsd.go
+25
View File
@@ -0,0 +1,25 @@
//go:build darwin || freebsd
package fuse
import (
"strings"
"syscall"
)
// fuseFilesystem reports whether path sits on a FUSE mount; macFUSE, osxfuse
// and fusefs all carry "fuse" in the filesystem type name.
func fuseFilesystem(path string) bool {
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return false
}
name := make([]byte, 0, len(st.Fstypename))
for _, c := range st.Fstypename {
if c == 0 {
break
}
name = append(name, byte(c))
}
return strings.Contains(string(name), "fuse")
}
test/fuse_integration/fuse_mount_linux.go
+17
View File
@@ -0,0 +1,17 @@
//go:build linux
package fuse
import "syscall"
// fuseSuperMagic is statfs f_type for fuse and fuseblk mounts.
const fuseSuperMagic = 0x65735546
// fuseFilesystem reports whether path sits on a FUSE mount.
func fuseFilesystem(path string) bool {
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return false
}
return st.Type == fuseSuperMagic
}
test/fuse_integration/fuse_mount_other.go
+9
View File
@@ -0,0 +1,9 @@
//go:build (unix && !linux && !darwin && !freebsd) || (!unix && !windows)
package fuse
// fuseFilesystem has no portable check on these platforms; the mount check
// and write verification in Setup stay the only gates.
func fuseFilesystem(path string) bool {
return true
}
test/fuse_integration/mmap_unix.go
+55
View File
@@ -0,0 +1,55 @@
//go:build unix
package fuse
import (
"syscall"
"golang.org/x/sys/unix"
)
// Memory mapping support for Unix-like systems
func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) {
return unix.Mmap(fd, offset, length, prot, flags)
}
func munmapFile(data []byte) error {
if len(data) == 0 {
return nil
}
return unix.Munmap(data)
}
func msyncFile(data []byte, flags int) error {
if len(data) == 0 {
return nil
}
return unix.Msync(data, flags)
}
func isMmapSupported() bool {
return true
}
// Memory protection flags
const (
PROT_READ = syscall.PROT_READ
PROT_WRITE = syscall.PROT_WRITE
PROT_EXEC = syscall.PROT_EXEC
PROT_NONE = syscall.PROT_NONE
)
// Memory mapping flags
const (
MAP_SHARED = syscall.MAP_SHARED
MAP_PRIVATE = syscall.MAP_PRIVATE
MAP_ANONYMOUS = syscall.MAP_ANON
)
// Memory sync flags
const (
MS_ASYNC = syscall.MS_ASYNC
MS_SYNC = syscall.MS_SYNC
MS_INVALIDATE = syscall.MS_INVALIDATE
)
test/fuse_integration/mmap_unsupported.go
+47
View File
@@ -0,0 +1,47 @@
//go:build !unix
package fuse
import (
"errors"
)
// Memory mapping support for unsupported platforms
var ErrMmapNotSupported = errors.New("memory mapping not supported on this platform")
func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) {
return nil, ErrMmapNotSupported
}
func munmapFile(data []byte) error {
return ErrMmapNotSupported
}
func msyncFile(data []byte, flags int) error {
return ErrMmapNotSupported
}
func isMmapSupported() bool {
return false
}
// Dummy constants for unsupported platforms
const (
PROT_READ = 0x1
PROT_WRITE = 0x2
PROT_EXEC = 0x4
PROT_NONE = 0x0
)
const (
MAP_SHARED = 0x01
MAP_PRIVATE = 0x02
MAP_ANONYMOUS = 0x20
)
const (
MS_ASYNC = 0x1
MS_SYNC = 0x4
MS_INVALIDATE = 0x2
)
test/fuse_integration/mounted_linux_test.go
+27
View File
@@ -0,0 +1,27 @@
//go:build linux
package fuse
import "testing"
func TestMountedAt(t *testing.T) {
for _, p := range []string{"/proc", "/proc/"} {
if !mountedAt(p) {
t.Errorf("mountedAt(%q) = false, want true", p)
}
}
plain := t.TempDir()
for _, p := range []string{plain, plain + "/"} {
if mountedAt(p) {
t.Errorf("mountedAt(%q) = true, want false", p)
}
}
}
func TestFuseFilesystem(t *testing.T) {
for _, p := range []string{"/proc", t.TempDir()} {
if fuseFilesystem(p) {
t.Errorf("fuseFilesystem(%q) = true, want false", p)
}
}
}
test/fuse_integration/mounted_unix.go
+33
View File
@@ -0,0 +1,33 @@
//go:build unix
package fuse
import (
"path/filepath"
"syscall"
)
// deviceID returns the device ID of the filesystem containing path.
func deviceID(path string) (uint64, error) {
var st syscall.Stat_t
if err := syscall.Stat(path, &st); err != nil {
return 0, err
}
return uint64(st.Dev), nil
}
// mountedAt reports whether something is actually mounted at path: a plain
// directory shares its parent's device, a mount point does not. The path is
// cleaned first so a trailing slash does not make Dir return the mount itself.
func mountedAt(path string) bool {
path = filepath.Clean(path)
dev, err := deviceID(path)
if err != nil {
return false
}
parentDev, err := deviceID(filepath.Dir(path))
if err != nil {
return false
}
return dev != parentDev
}
test/fuse_integration/mounted_unsupported.go
+16
View File
@@ -0,0 +1,16 @@
//go:build !unix && !windows
package fuse
// FUSE mounts are unix-only: reporting one device keeps waitForMount from
// mistaking the plain pre-mount directory for a live mount on platforms
// where a mount can never appear.
func deviceID(path string) (uint64, error) {
return 0, nil
}
// mountedAt cannot be detected off unix; report mounted so an external
// TEST_MOUNT_POINT keeps its best-effort write verification.
func mountedAt(path string) bool {
return true
}
test/fuse_integration/mounted_windows.go
+40
View File
@@ -0,0 +1,40 @@
//go:build windows
package fuse
import (
"os"
"path/filepath"
"syscall"
)
// deviceID has no portable meaning here: reporting one device keeps
// waitForMount from mistaking the plain pre-mount directory for a live mount.
func deviceID(path string) (uint64, error) {
return 0, nil
}
// mountedAt reports whether path is a WinFsp mount: a directory mount is a
// reparse point and a drive mount is a volume root, while a plain directory
// is neither.
func mountedAt(path string) bool {
abs, err := filepath.Abs(path)
if err != nil {
return false
}
if filepath.Dir(abs) == abs {
return true
}
info, err := os.Lstat(abs)
if err != nil {
return false
}
data, ok := info.Sys().(*syscall.Win32FileAttributeData)
return ok && data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0
}
// fuseFilesystem is covered by the WinFsp checks in mountedAt: WinFsp is the
// only FUSE provider on the platform.
func fuseFilesystem(path string) bool {
return true
}
test/fuse_integration/posix_Makefile
+531
View File
@@ -0,0 +1,531 @@
# SeaweedFS POSIX Compliance Testing Makefile
SHELL := /bin/bash
# go test is piped through tee everywhere; without pipefail a failing suite
# still exits 0 because only tee's status propagates.
.SHELLFLAGS := -o pipefail -c
# Configuration
WEED_BINARY := $(shell which weed 2>/dev/null || echo "../../weed")
GO_VERSION := 1.21
TEST_TIMEOUT := 45m
COVERAGE_FILE := posix_coverage.out
REPORT_DIR := reports
EXTERNAL_TOOLS_DIR := external_tools
MOUNT_POINT ?= /tmp/seaweedfs_mount
ifneq ($(TEST_MOUNT_POINT),)
MOUNT_POINT := $(TEST_MOUNT_POINT)
endif
# Go test command prefix for external mount points
GO_TEST_PREFIX :=
ifneq ($(TEST_MOUNT_POINT),)
GO_TEST_PREFIX := TEST_MOUNT_POINT="$(TEST_MOUNT_POINT)" TEST_SKIP_CLUSTER_SETUP="true"
endif
# Test categories
POSIX_BASIC_TESTS := posix_compliance_test.go
POSIX_EXTENDED_TESTS := posix_extended_test.go
POSIX_EXTERNAL_TESTS := posix_external_test.go
# -f is not propagated through MAKEFLAGS, so recursive calls must name this
# file explicitly or the sub-make reads the default Makefile instead.
MAKE_SELF := $(MAKE) -f $(firstword $(MAKEFILE_LIST))
# Colors for output — bash echo does not expand \033, so bake the escapes in
RED := $(shell printf '\033[31m')
GREEN := $(shell printf '\033[32m')
YELLOW := $(shell printf '\033[33m')
BLUE := $(shell printf '\033[34m')
MAGENTA := $(shell printf '\033[35m')
CYAN := $(shell printf '\033[36m')
WHITE := $(shell printf '\033[37m')
RESET := $(shell printf '\033[0m')
.DEFAULT_GOAL := help
# Prerequisites checks
check-binary:
@if [ ! -f "$(WEED_BINARY)" ]; then \
echo "$(RED)❌ SeaweedFS binary not found at $(WEED_BINARY)$(RESET)"; \
echo " Please run 'make' in the root directory first"; \
exit 1; \
fi
@echo "$(GREEN)[OK] SeaweedFS binary found at $(WEED_BINARY)$(RESET)"
check-fuse:
@if command -v fusermount >/dev/null 2>&1; then \
echo "$(GREEN)[OK] FUSE is installed (Linux)$(RESET)"; \
elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \
echo "$(GREEN)[OK] FUSE is available (macOS)$(RESET)"; \
else \
echo "$(RED)❌ FUSE not found. Please install:$(RESET)"; \
echo " Ubuntu/Debian: sudo apt-get install fuse"; \
echo " CentOS/RHEL: sudo yum install fuse"; \
echo " macOS: brew install macfuse"; \
exit 1; \
fi
check-go:
@go version | grep -Eq "go(1\.(2[1-9]|[3-9][0-9]+)|[2-9]\.)" || \
(echo "$(RED)❌ Go $(GO_VERSION)+ required. Current: $$(go version)$(RESET)" && exit 1)
@echo "$(GREEN)[OK] Go version check passed$(RESET)"
check-prereqs: check-go check-fuse check-binary
@echo "$(GREEN)[OK] All prerequisites satisfied$(RESET)"
# Setup and initialization
setup-reports:
@mkdir -p $(REPORT_DIR)
@mkdir -p $(EXTERNAL_TOOLS_DIR)
setup-external-tools: setup-reports
@echo "$(BLUE)🛠️ Setting up external POSIX test tools...$(RESET)"
@$(MAKE_SELF) setup-pjdfstest || echo "$(YELLOW)[WARNING] pjdfstest setup failed - continuing without it$(RESET)"
@$(MAKE_SELF) setup-nfstest || echo "$(YELLOW)[WARNING] nfstest setup failed - continuing without it$(RESET)"
@$(MAKE_SELF) setup-fio || echo "$(YELLOW)[WARNING] FIO setup failed - continuing without it$(RESET)"
# External tools setup
setup-pjdfstest:
@if [ ! -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \
echo "$(BLUE)[SETUP] Setting up pjdfstest...$(RESET)"; \
if ! command -v autoreconf >/dev/null 2>&1; then \
echo "$(YELLOW)[WARNING] autoreconf not found - installing build-essential...$(RESET)"; \
if command -v apt-get >/dev/null 2>&1; then \
sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool; \
elif command -v yum >/dev/null 2>&1; then \
sudo yum install -y gcc make autoconf automake libtool; \
else \
echo "$(YELLOW)[WARNING] Please install build tools manually$(RESET)"; \
exit 1; \
fi; \
fi; \
cd $(EXTERNAL_TOOLS_DIR) && \
git clone https://github.com/pjd/pjdfstest.git && \
cd pjdfstest && \
autoreconf -ifs && \
./configure && \
make; \
else \
echo "$(GREEN)[OK] pjdfstest already setup$(RESET)"; \
fi
setup-nfstest:
@echo "$(BLUE)[INSTALL] Installing nfstest...$(RESET)"
@pip3 install --user nfstest || \
echo "$(YELLOW)[WARNING] nfstest installation failed. Install manually: pip3 install nfstest$(RESET)"
setup-fio:
@if ! command -v fio >/dev/null 2>&1; then \
echo "$(BLUE)[SETUP] Installing FIO...$(RESET)"; \
if command -v apt-get >/dev/null 2>&1; then \
sudo apt-get update && sudo apt-get install -y fio; \
elif command -v yum >/dev/null 2>&1; then \
sudo yum install -y fio; \
elif command -v brew >/dev/null 2>&1; then \
brew install fio; \
else \
echo "$(YELLOW)[WARNING] Please install FIO manually$(RESET)"; \
fi; \
else \
echo "$(GREEN)[OK] FIO already installed$(RESET)"; \
fi
# Core test execution
test-posix-basic: check-prereqs setup-reports
@echo "$(CYAN)[TEST] Running basic POSIX compliance tests...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXCompliance . 2>&1 | \
tee $(REPORT_DIR)/posix_basic_results.log
test-posix-extended: check-prereqs setup-reports
@echo "$(CYAN)[TEST] Running extended POSIX compliance tests...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXExtended . 2>&1 | \
tee $(REPORT_DIR)/posix_extended_results.log
test-posix-external: check-prereqs setup-reports
@echo "$(CYAN)[TEST] Running external POSIX test suite integration...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \
echo "$(GREEN)[OK] External tools available - running external tests$(RESET)"; \
$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestExternalPOSIXSuites . 2>&1 | \
tee $(REPORT_DIR)/posix_external_results.log; \
else \
echo "$(YELLOW)[WARNING] External tools not available - skipping external tests$(RESET)"; \
echo "External tests skipped - tools not available" > $(REPORT_DIR)/posix_external_results.log; \
fi
# Comprehensive test suites
test-posix-full: test-posix-basic test-posix-extended test-posix-external
@echo "$(GREEN)[OK] Full POSIX compliance test suite completed$(RESET)"
@$(MAKE_SELF) generate-report
test-posix-critical: check-prereqs setup-reports
@echo "$(CYAN)[TEST] Running critical POSIX compliance tests...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout 15m -failfast=false \
-run "TestPOSIXCompliance/(FileOperations|DirectoryOperations|PermissionTests|IOOperations)" \
. 2>&1 | tee $(REPORT_DIR)/posix_critical_results.log
test-posix-stress: check-prereqs setup-reports
@echo "$(CYAN)[TEST] Running POSIX stress tests...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \
echo "$(GREEN)[OK] External tools available - running stress tests$(RESET)"; \
$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false \
-run "TestExternalPOSIXSuites/CustomPOSIXTests" \
. 2>&1 | tee $(REPORT_DIR)/posix_stress_results.log; \
else \
echo "$(YELLOW)[WARNING] External tools not available - skipping stress tests$(RESET)"; \
echo "Stress tests skipped - tools not available" > $(REPORT_DIR)/posix_stress_results.log; \
fi
# Performance and benchmarks
benchmark-posix: check-prereqs setup-reports
@echo "$(CYAN)📈 Running POSIX performance benchmarks...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -run '^$$' -bench=. -benchmem \
. 2>&1 | \
tee $(REPORT_DIR)/posix_benchmark_results.log
profile-posix: check-prereqs setup-reports
@echo "$(CYAN)[PROFILE] Running POSIX tests with profiling...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -cpuprofile $(REPORT_DIR)/posix.cpu.prof \
-memprofile $(REPORT_DIR)/posix.mem.prof -run TestPOSIXCompliance .
@echo "$(GREEN)[PROFILE] Profiles generated:$(RESET)"
@echo " CPU: $(REPORT_DIR)/posix.cpu.prof"
@echo " Memory: $(REPORT_DIR)/posix.mem.prof"
@echo "$(BLUE)View with: go tool pprof $(REPORT_DIR)/posix.cpu.prof$(RESET)"
# Coverage analysis
coverage-posix: check-prereqs setup-reports
@echo "$(CYAN)[COVERAGE] Running POSIX tests with coverage analysis...$(RESET)"
ifneq ($(TEST_MOUNT_POINT),)
@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
endif
@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -coverprofile=$(REPORT_DIR)/$(COVERAGE_FILE) \
.
@go tool cover -html=$(REPORT_DIR)/$(COVERAGE_FILE) -o $(REPORT_DIR)/posix_coverage.html
@echo "$(GREEN)[COVERAGE] Coverage report generated: $(REPORT_DIR)/posix_coverage.html$(RESET)"
# The external suites exercise whatever filesystem MOUNT_POINT resolves to;
# mkdir -p would happily point them at a plain local directory and report a
# pass the mount never earned.
require-mounted:
@mnt="$(MOUNT_POINT)"; mnt="$${mnt%/}"; \
if ! mountpoint -q "$$mnt" 2>/dev/null && ! mount | grep -qF " $$mnt "; then \
echo "$(RED)❌ $$mnt is not a mounted filesystem - refusing to run against the local disk$(RESET)"; \
exit 1; \
fi
# External tool tests
test-pjdfstest: setup-pjdfstest setup-reports require-mounted
@echo "$(CYAN)[TEST] Running pjdfstest suite...$(RESET)"
@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \
mkdir -p $(MOUNT_POINT)/pjdfstest_workdir; \
cd $(MOUNT_POINT)/pjdfstest_workdir && \
prove -r $(CURDIR)/$(EXTERNAL_TOOLS_DIR)/pjdfstest/tests/ 2>&1 | tee $(CURDIR)/$(REPORT_DIR)/pjdfstest_results.log; \
else \
echo "$(RED)❌ pjdfstest not setup$(RESET)"; \
exit 1; \
fi
test-nfstest-posix: setup-nfstest setup-reports require-mounted
@echo "$(CYAN)[TEST] Running nfstest_posix...$(RESET)"
@if command -v nfstest_posix >/dev/null 2>&1; then \
mkdir -p $(MOUNT_POINT); \
nfstest_posix --path $(MOUNT_POINT) --verbose 2>&1 | \
tee $(REPORT_DIR)/nfstest_results.log; \
else \
echo "$(YELLOW)[WARNING] nfstest_posix not available$(RESET)"; \
fi
# FIO-based performance tests
test-fio-posix: setup-reports require-mounted
@echo "$(CYAN)[TEST] Running FIO-based POSIX I/O tests...$(RESET)"
@$(MAKE_SELF) create-fio-configs
@if command -v fio >/dev/null 2>&1; then \
mkdir -p $(MOUNT_POINT); \
for config in $(REPORT_DIR)/fio_*.conf; do \
echo "$(BLUE)Running FIO config: $$config$(RESET)"; \
fio $$config --output=$(REPORT_DIR)/$$(basename $$config .conf)_results.log; \
done; \
else \
echo "$(YELLOW)[WARNING] FIO not available$(RESET)"; \
fi
create-fio-configs: setup-reports
@echo "$(BLUE)[CONFIG] Creating FIO test configurations...$(RESET)"
@echo "[global]" > $(REPORT_DIR)/fio_random_rw.conf
@echo "name=posix_random_rw" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "ioengine=sync" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "iodepth=1" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "rw=randrw" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "bs=4k" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "direct=0" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "size=100m" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "numjobs=4" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "runtime=30" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "time_based" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "[random_rw_test]" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_random_rw.conf
@echo "[global]" > $(REPORT_DIR)/fio_sequential.conf
@echo "name=posix_sequential" >> $(REPORT_DIR)/fio_sequential.conf
@echo "ioengine=sync" >> $(REPORT_DIR)/fio_sequential.conf
@echo "iodepth=1" >> $(REPORT_DIR)/fio_sequential.conf
@echo "bs=1m" >> $(REPORT_DIR)/fio_sequential.conf
@echo "direct=0" >> $(REPORT_DIR)/fio_sequential.conf
@echo "size=500m" >> $(REPORT_DIR)/fio_sequential.conf
@echo "runtime=60" >> $(REPORT_DIR)/fio_sequential.conf
@echo "time_based" >> $(REPORT_DIR)/fio_sequential.conf
@echo "" >> $(REPORT_DIR)/fio_sequential.conf
@echo "[seq_write]" >> $(REPORT_DIR)/fio_sequential.conf
@echo "rw=write" >> $(REPORT_DIR)/fio_sequential.conf
@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf
@echo "" >> $(REPORT_DIR)/fio_sequential.conf
@echo "[seq_read]" >> $(REPORT_DIR)/fio_sequential.conf
@echo "rw=read" >> $(REPORT_DIR)/fio_sequential.conf
@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf
# Reporting and analysis
generate-report: setup-reports
@echo "$(BLUE)[REPORT] Generating comprehensive POSIX compliance report...$(RESET)"
@$(MAKE_SELF) generate-html-report
@$(MAKE_SELF) generate-text-report
@$(MAKE_SELF) generate-json-report
generate-html-report:
@echo "$(BLUE)[REPORT] Generating HTML report...$(RESET)"
@echo "<!DOCTYPE html>" > $(REPORT_DIR)/posix_compliance_report.html
@echo "<html><head><title>SeaweedFS POSIX Compliance Report</title>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "<style>body{font-family:Arial;margin:20px}.pass{color:green}.fail{color:red}.summary{background:#f5f5f5;padding:10px;margin:10px 0}.test-section{margin:20px 0;border-left:3px solid #ddd;padding-left:15px}</style>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "</head><body><h1>SeaweedFS POSIX Compliance Report</h1>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "<p>Generated on: $$(date)</p>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "<div class='summary'>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "<h2>Test Summary</h2>" >> $(REPORT_DIR)/posix_compliance_report.html
@if [ -f "$(REPORT_DIR)/posix_basic_results.log" ]; then \
TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
echo "<p>Basic Tests: <span class='pass'>$$PASSED passed</span>, <span class='fail'>$$FAILED failed</span></p>" >> $(REPORT_DIR)/posix_compliance_report.html; \
fi
@if [ -f "$(REPORT_DIR)/posix_critical_results.log" ]; then \
TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
echo "<p>Critical Tests: <span class='pass'>$$PASSED passed</span>, <span class='fail'>$$FAILED failed</span></p>" >> $(REPORT_DIR)/posix_compliance_report.html; \
fi
@echo "</div>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "<h2>Detailed Results</h2>" >> $(REPORT_DIR)/posix_compliance_report.html
@for logfile in $(REPORT_DIR)/*.log; do \
if [ -f "$$logfile" ]; then \
basename=$$(basename "$$logfile" .log); \
echo "<div class='test-section'>" >> $(REPORT_DIR)/posix_compliance_report.html; \
echo "<h3>$$basename</h3>" >> $(REPORT_DIR)/posix_compliance_report.html; \
echo "<pre>" >> $(REPORT_DIR)/posix_compliance_report.html; \
tail -50 "$$logfile" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' >> $(REPORT_DIR)/posix_compliance_report.html; \
echo "</pre></div>" >> $(REPORT_DIR)/posix_compliance_report.html; \
fi; \
done
@echo "</body></html>" >> $(REPORT_DIR)/posix_compliance_report.html
@echo "$(GREEN)[OK] HTML report generated: $(REPORT_DIR)/posix_compliance_report.html$(RESET)"
generate-text-report:
@echo "$(BLUE)[REPORT] Generating text report...$(RESET)"
@echo "SeaweedFS POSIX Compliance Report" > $(REPORT_DIR)/posix_compliance_summary.txt
@echo "=================================" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "Generated: $$(date)" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "SeaweedFS Version: $$($(WEED_BINARY) version 2>/dev/null || echo 'Unknown')" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "Test Summary: See individual log files for detailed results" >> $(REPORT_DIR)/posix_compliance_summary.txt
@echo "$(GREEN)[OK] Text report generated: $(REPORT_DIR)/posix_compliance_summary.txt$(RESET)"
generate-json-report:
@echo "$(BLUE)[REPORT] Generating JSON report...$(RESET)"
@echo "{" > $(REPORT_DIR)/posix_compliance_report.json
@echo " \"report_type\": \"posix_compliance\"," >> $(REPORT_DIR)/posix_compliance_report.json
@echo " \"timestamp\": \"$$(date -u +"%Y-%m-%dT%H:%M:%SZ")\"," >> $(REPORT_DIR)/posix_compliance_report.json
@echo " \"seaweedfs_version\": \"$$($(WEED_BINARY) version 2>/dev/null | head -n1 | sed 's/\\/\\\\/g; s/\"/\\\"/g' || echo 'Unknown')\"," >> $(REPORT_DIR)/posix_compliance_report.json
@echo " \"test_environment\": { \"os\": \"$$(uname -s)\", \"arch\": \"$$(uname -m)\" }," >> $(REPORT_DIR)/posix_compliance_report.json
@echo " \"test_suites\": {" >> $(REPORT_DIR)/posix_compliance_report.json
@FIRST=true; \
for logfile in $(REPORT_DIR)/*.log; do \
if [ -f "$$logfile" ]; then \
basename=$$(basename "$$logfile" .log); \
if [ "$$FIRST" = "true" ]; then \
FIRST=false; \
else \
echo " ," >> $(REPORT_DIR)/posix_compliance_report.json; \
fi; \
PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
TOTAL=$$(expr "$$PASSED" + "$$FAILED" 2>/dev/null || echo 0); \
echo " \"$$basename\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"total_tests\": $$TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"passed\": $$PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"failed\": $$FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \
if [ "$$TOTAL" -gt 0 ] 2>/dev/null; then \
SUCCESS_RATE=$$(awk "BEGIN {printf \"%.2f\", ($$PASSED/$$TOTAL)*100}" 2>/dev/null || echo "0.00"); \
else \
SUCCESS_RATE="0.00"; \
fi; \
echo " \"success_rate\": \"$$SUCCESS_RATE%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " }" >> $(REPORT_DIR)/posix_compliance_report.json; \
fi; \
done
@echo " }," >> $(REPORT_DIR)/posix_compliance_report.json
@TOTAL_PASSED=0; TOTAL_FAILED=0; \
for logfile in $(REPORT_DIR)/*.log; do \
if [ -f "$$logfile" ]; then \
PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
TOTAL_PASSED=$$(expr "$$TOTAL_PASSED" + "$$PASSED" 2>/dev/null || echo "$$TOTAL_PASSED"); \
TOTAL_FAILED=$$(expr "$$TOTAL_FAILED" + "$$FAILED" 2>/dev/null || echo "$$TOTAL_FAILED"); \
fi; \
done; \
GRAND_TOTAL=$$(expr "$$TOTAL_PASSED" + "$$TOTAL_FAILED" 2>/dev/null || echo 0); \
if [ "$$GRAND_TOTAL" -gt 0 ] 2>/dev/null; then \
OVERALL_SUCCESS=$$(awk "BEGIN {printf \"%.2f\", ($$TOTAL_PASSED/$$GRAND_TOTAL)*100}" 2>/dev/null || echo "0.00"); \
else \
OVERALL_SUCCESS="0.00"; \
fi; \
echo " \"summary\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"total_tests\": $$GRAND_TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"total_passed\": $$TOTAL_PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"total_failed\": $$TOTAL_FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " \"overall_success_rate\": \"$$OVERALL_SUCCESS%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \
echo " }" >> $(REPORT_DIR)/posix_compliance_report.json
@echo "}" >> $(REPORT_DIR)/posix_compliance_report.json
@echo "$(GREEN)[OK] JSON report generated: $(REPORT_DIR)/posix_compliance_report.json$(RESET)"
# Cleanup and maintenance
clean:
@echo "$(YELLOW)🧹 Cleaning up test artifacts...$(RESET)"
@rm -rf $(REPORT_DIR)
@rm -rf /tmp/seaweedfs_*_test_*
@go clean -testcache
@echo "$(GREEN)[OK] Cleanup complete$(RESET)"
clean-external-tools:
@echo "$(YELLOW)🧹 Cleaning up external tools...$(RESET)"
@rm -rf $(EXTERNAL_TOOLS_DIR)
clean-all: clean clean-external-tools
# Development and debugging
validate:
@echo "$(BLUE)[OK] Validating test files...$(RESET)"
@go build -o /dev/null ./...
@echo "$(GREEN)[OK] All test files compile successfully$(RESET)"
fmt:
@echo "$(BLUE)🎨 Formatting Go code...$(RESET)"
@go fmt ./...
lint:
@echo "$(BLUE)🔍 Running linter...$(RESET)"
@if command -v golangci-lint >/dev/null 2>&1; then \
golangci-lint run; \
else \
echo "$(YELLOW)[WARNING] golangci-lint not found, running go vet instead$(RESET)"; \
go vet ./...; \
fi
# CI/CD integration
ci-posix-tests: check-prereqs setup-external-tools
@echo "$(CYAN)🚀 Running POSIX tests for CI/CD...$(RESET)"
@$(MAKE_SELF) test-posix-critical
@$(MAKE_SELF) generate-report
# Documentation and help
list-tests:
@echo "$(BLUE)📋 Available POSIX test functions:$(RESET)"
@grep -r "^func Test" *.go 2>/dev/null | sed 's/.*func \(Test[^(]*\).*/ \1/' | sort || echo "No test files found"
test-info:
@echo "$(BLUE)[INFO] POSIX Test Information:$(RESET)"
@echo "Test files:"
@echo " - $(POSIX_BASIC_TESTS): Core POSIX compliance tests"
@echo " - $(POSIX_EXTENDED_TESTS): Extended POSIX feature tests"
@echo " - $(POSIX_EXTERNAL_TESTS): External test suite integration"
@echo ""
@echo "External tools:"
@echo " - pjdfstest: Comprehensive POSIX filesystem test suite"
@echo " - nfstest: Network filesystem POSIX API verification"
@echo " - FIO: Flexible I/O performance testing"
help:
@echo "$(CYAN)SeaweedFS POSIX Compliance Testing$(RESET)"
@echo "=================================="
@echo ""
@echo "$(WHITE)Prerequisites:$(RESET)"
@echo " $(GREEN)make check-prereqs$(RESET) - Check all prerequisites"
@echo " $(GREEN)make setup-external-tools$(RESET) - Setup external POSIX test tools"
@echo ""
@echo "$(WHITE)Core POSIX Tests:$(RESET)"
@echo " $(GREEN)make test-posix-basic$(RESET) - Run basic POSIX compliance tests"
@echo " $(GREEN)make test-posix-extended$(RESET) - Run extended POSIX feature tests"
@echo " $(GREEN)make test-posix-external$(RESET) - Run external test suite integration"
@echo " $(GREEN)make test-posix-full$(RESET) - Run complete POSIX test suite"
@echo ""
@echo "$(WHITE)Focused Tests:$(RESET)"
@echo " $(GREEN)make test-posix-critical$(RESET) - Run critical POSIX compliance tests"
@echo " $(GREEN)make test-posix-stress$(RESET) - Run POSIX stress tests"
@echo ""
@echo "$(WHITE)Performance & Analysis:$(RESET)"
@echo " $(GREEN)make benchmark-posix$(RESET) - Run POSIX performance benchmarks"
@echo " $(GREEN)make profile-posix$(RESET) - Run tests with performance profiling"
@echo " $(GREEN)make coverage-posix$(RESET) - Run tests with coverage analysis"
@echo " $(GREEN)make test-fio-posix$(RESET) - Run FIO-based I/O performance tests"
@echo ""
@echo "$(WHITE)External Test Suites:$(RESET)"
@echo " $(GREEN)make test-pjdfstest$(RESET) - Run pjdfstest suite"
@echo " $(GREEN)make test-nfstest-posix$(RESET) - Run nfstest_posix"
@echo ""
@echo "$(WHITE)Reporting:$(RESET)"
@echo " $(GREEN)make generate-report$(RESET) - Generate comprehensive compliance report"
@echo ""
@echo "$(WHITE)Development:$(RESET)"
@echo " $(GREEN)make validate$(RESET) - Validate test file compilation"
@echo " $(GREEN)make fmt$(RESET) - Format Go code"
@echo " $(GREEN)make lint$(RESET) - Run linter"
@echo " $(GREEN)make list-tests$(RESET) - List available test functions"
@echo ""
@echo "$(WHITE)Maintenance:$(RESET)"
@echo " $(GREEN)make clean$(RESET) - Clean up test artifacts"
@echo " $(GREEN)make clean-all$(RESET) - Clean everything including external tools"
@echo ""
@echo "$(WHITE)CI/CD Integration:$(RESET)"
@echo " $(GREEN)make ci-posix-tests$(RESET) - Run POSIX tests optimized for CI/CD"
.PHONY: help check-prereqs check-binary check-fuse check-go setup-reports \
setup-external-tools setup-pjdfstest setup-nfstest setup-fio \
test-posix-basic test-posix-extended test-posix-external test-posix-full \
test-posix-critical test-posix-stress benchmark-posix profile-posix coverage-posix \
test-pjdfstest test-nfstest-posix test-fio-posix create-fio-configs \
generate-report generate-html-report generate-text-report generate-json-report \
clean clean-external-tools clean-all validate fmt lint ci-posix-tests \
list-tests test-info
test/fuse_integration/posix_compliance_test.go
+713
View File
@@ -0,0 +1,713 @@
//go:build !windows
package fuse
import (
"fmt"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// POSIXComplianceTestSuite provides comprehensive POSIX compliance testing for FUSE mounts
type POSIXComplianceTestSuite struct {
framework *FuseTestFramework
t *testing.T
}
// NewPOSIXComplianceTestSuite creates a new POSIX compliance test suite
func NewPOSIXComplianceTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXComplianceTestSuite {
return &POSIXComplianceTestSuite{
framework: framework,
t: t,
}
}
// TestPOSIXCompliance runs all POSIX compliance tests
func TestPOSIXCompliance(t *testing.T) {
config := DefaultTestConfig()
config.EnableDebug = true
config.MountOptions = []string{"-allowOthers", "-nonempty"}
framework := NewFuseTestFramework(t, config)
defer framework.Cleanup()
require.NoError(t, framework.Setup(config))
suite := NewPOSIXComplianceTestSuite(t, framework)
// Run all POSIX compliance test categories
t.Run("FileOperations", suite.TestFileOperations)
t.Run("DirectoryOperations", suite.TestDirectoryOperations)
t.Run("SymlinkOperations", suite.TestSymlinkOperations)
t.Run("PermissionTests", suite.TestPermissions)
t.Run("TimestampTests", suite.TestTimestamps)
t.Run("IOOperations", suite.TestIOOperations)
t.Run("FileDescriptorTests", suite.TestFileDescriptors)
t.Run("AtomicOperations", suite.TestAtomicOperations)
t.Run("ConcurrentAccess", suite.TestConcurrentAccess)
t.Run("ErrorHandling", suite.TestErrorHandling)
}
// TestFileOperations tests POSIX file operation compliance
func (s *POSIXComplianceTestSuite) TestFileOperations(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("CreateFile", func(t *testing.T) {
filepath := filepath.Join(mountPoint, "test_create.txt")
// Test file creation with O_CREAT
fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_WRONLY, 0644)
require.NoError(t, err)
require.Greater(t, fd, 0)
err = syscall.Close(fd)
require.NoError(t, err)
// Verify file exists
_, err = os.Stat(filepath)
require.NoError(t, err)
})
t.Run("CreateExclusiveFile", func(t *testing.T) {
filepath := filepath.Join(mountPoint, "test_excl.txt")
// First creation should succeed
fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644)
require.NoError(t, err)
syscall.Close(fd)
// Second creation should fail with EEXIST
_, err = syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644)
require.Error(t, err)
require.Equal(t, syscall.EEXIST, err)
})
t.Run("TruncateFile", func(t *testing.T) {
filepath := filepath.Join(mountPoint, "test_truncate.txt")
content := []byte("Hello, World! This is a test file for truncation.")
// Create file with content
err := os.WriteFile(filepath, content, 0644)
require.NoError(t, err)
// Truncate to 5 bytes
err = syscall.Truncate(filepath, 5)
require.NoError(t, err)
// Verify truncation
readContent, err := os.ReadFile(filepath)
require.NoError(t, err)
require.Equal(t, []byte("Hello"), readContent)
})
t.Run("UnlinkFile", func(t *testing.T) {
filepath := filepath.Join(mountPoint, "test_unlink.txt")
// Create file
err := os.WriteFile(filepath, []byte("test"), 0644)
require.NoError(t, err)
// Unlink file
err = syscall.Unlink(filepath)
require.NoError(t, err)
// Verify file no longer exists
_, err = os.Stat(filepath)
require.True(t, os.IsNotExist(err))
})
}
// TestDirectoryOperations tests POSIX directory operation compliance
func (s *POSIXComplianceTestSuite) TestDirectoryOperations(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("CreateDirectory", func(t *testing.T) {
dirPath := filepath.Join(mountPoint, "test_mkdir")
err := syscall.Mkdir(dirPath, 0755)
require.NoError(t, err)
// Verify directory exists and has correct type
stat, err := os.Stat(dirPath)
require.NoError(t, err)
require.True(t, stat.IsDir())
})
t.Run("RemoveDirectory", func(t *testing.T) {
dirPath := filepath.Join(mountPoint, "test_rmdir")
// Create directory
err := os.Mkdir(dirPath, 0755)
require.NoError(t, err)
// Remove directory
err = syscall.Rmdir(dirPath)
require.NoError(t, err)
// Verify directory no longer exists
_, err = os.Stat(dirPath)
require.True(t, os.IsNotExist(err))
})
t.Run("RemoveNonEmptyDirectory", func(t *testing.T) {
dirPath := filepath.Join(mountPoint, "test_rmdir_nonempty")
filePath := filepath.Join(dirPath, "file.txt")
// Create directory and file
err := os.Mkdir(dirPath, 0755)
require.NoError(t, err)
err = os.WriteFile(filePath, []byte("test"), 0644)
require.NoError(t, err)
// Attempt to remove non-empty directory should fail
err = syscall.Rmdir(dirPath)
require.Error(t, err)
require.Equal(t, syscall.ENOTEMPTY, err)
})
t.Run("RenameDirectory", func(t *testing.T) {
oldPath := filepath.Join(mountPoint, "old_dir")
newPath := filepath.Join(mountPoint, "new_dir")
// Create directory
err := os.Mkdir(oldPath, 0755)
require.NoError(t, err)
// Rename directory
err = os.Rename(oldPath, newPath)
require.NoError(t, err)
// Verify old path doesn't exist and new path does
_, err = os.Stat(oldPath)
require.True(t, os.IsNotExist(err))
stat, err := os.Stat(newPath)
require.NoError(t, err)
require.True(t, stat.IsDir())
})
}
// TestSymlinkOperations tests POSIX symlink operation compliance
func (s *POSIXComplianceTestSuite) TestSymlinkOperations(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("CreateSymlink", func(t *testing.T) {
targetFile := filepath.Join(mountPoint, "target.txt")
linkFile := filepath.Join(mountPoint, "link.txt")
// Create target file
err := os.WriteFile(targetFile, []byte("target content"), 0644)
require.NoError(t, err)
// Create symlink
err = os.Symlink(targetFile, linkFile)
require.NoError(t, err)
// Verify symlink properties
linkStat, err := os.Lstat(linkFile)
require.NoError(t, err)
require.Equal(t, os.ModeSymlink, linkStat.Mode()&os.ModeType)
// Verify symlink content
linkTarget, err := os.Readlink(linkFile)
require.NoError(t, err)
require.Equal(t, targetFile, linkTarget)
// Verify following symlink works
content, err := os.ReadFile(linkFile)
require.NoError(t, err)
require.Equal(t, []byte("target content"), content)
})
t.Run("BrokenSymlink", func(t *testing.T) {
nonexistentTarget := filepath.Join(mountPoint, "nonexistent.txt")
linkFile := filepath.Join(mountPoint, "broken_link.txt")
// Create symlink to nonexistent file
err := os.Symlink(nonexistentTarget, linkFile)
require.NoError(t, err)
// Lstat should work (doesn't follow symlink)
_, err = os.Lstat(linkFile)
require.NoError(t, err)
// Stat should fail (follows symlink to nonexistent target)
_, err = os.Stat(linkFile)
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
}
// TestPermissions tests POSIX permission compliance
func (s *POSIXComplianceTestSuite) TestPermissions(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("FilePermissions", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "perm_test.txt")
// Create file with specific permissions
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0642)
require.NoError(t, err)
syscall.Close(fd)
// Verify permissions
stat, err := os.Stat(testFile)
require.NoError(t, err)
// Note: The final file permissions are affected by the system's umask.
// We check against the requested mode and the mode as affected by a common umask (e.g. 0022).
actualMode := stat.Mode() & os.ModePerm
expectedMode := os.FileMode(0642)
expectedModeWithUmask := os.FileMode(0640) // e.g., 0642 with umask 0002 or 0022
// Accept either the exact permissions or permissions as modified by umask
if actualMode != expectedMode && actualMode != expectedModeWithUmask {
t.Errorf("Expected file permissions %o or %o, but got %o",
expectedMode, expectedModeWithUmask, actualMode)
}
})
t.Run("ChangeFilePermissions", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "chmod_test.txt")
// Create file
err := os.WriteFile(testFile, []byte("test"), 0644)
require.NoError(t, err)
// Change permissions
err = os.Chmod(testFile, 0755)
require.NoError(t, err)
// Verify new permissions
stat, err := os.Stat(testFile)
require.NoError(t, err)
require.Equal(t, os.FileMode(0755), stat.Mode()&os.ModePerm)
})
t.Run("DirectoryPermissions", func(t *testing.T) {
testDir := filepath.Join(mountPoint, "perm_dir")
// Create directory with specific permissions
err := syscall.Mkdir(testDir, 0750)
require.NoError(t, err)
// Verify permissions
stat, err := os.Stat(testDir)
require.NoError(t, err)
require.Equal(t, os.FileMode(0750)|os.ModeDir, stat.Mode())
})
}
// TestTimestamps tests POSIX timestamp compliance
func (s *POSIXComplianceTestSuite) TestTimestamps(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("AccessTime", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "atime_test.txt")
// Create file
err := os.WriteFile(testFile, []byte("test content"), 0644)
require.NoError(t, err)
// Get initial timestamps
stat1, err := os.Stat(testFile)
require.NoError(t, err)
// Wait a bit to ensure time difference
time.Sleep(100 * time.Millisecond)
// Read file (should update access time)
_, err = os.ReadFile(testFile)
require.NoError(t, err)
// Get new timestamps
stat2, err := os.Stat(testFile)
require.NoError(t, err)
// Access time should have been updated, and modification time should be unchanged.
stat1Sys := stat1.Sys().(*syscall.Stat_t)
stat2Sys := stat2.Sys().(*syscall.Stat_t)
require.True(t, getAtimeNano(stat2Sys) >= getAtimeNano(stat1Sys), "access time should be updated or stay the same")
require.Equal(t, stat1.ModTime(), stat2.ModTime(), "modification time should not change on read")
})
t.Run("ModificationTime", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "mtime_test.txt")
// Create file
err := os.WriteFile(testFile, []byte("initial content"), 0644)
require.NoError(t, err)
// Get initial timestamp
stat1, err := os.Stat(testFile)
require.NoError(t, err)
// Wait to ensure time difference
time.Sleep(100 * time.Millisecond)
// Modify file
err = os.WriteFile(testFile, []byte("modified content"), 0644)
require.NoError(t, err)
// Get new timestamp
stat2, err := os.Stat(testFile)
require.NoError(t, err)
// Modification time should have changed
require.True(t, stat2.ModTime().After(stat1.ModTime()))
})
t.Run("SetTimestamps", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "utime_test.txt")
// Create file
err := os.WriteFile(testFile, []byte("test"), 0644)
require.NoError(t, err)
// Set specific timestamps
atime := time.Now().Add(-24 * time.Hour)
mtime := time.Now().Add(-12 * time.Hour)
err = os.Chtimes(testFile, atime, mtime)
require.NoError(t, err)
// Verify timestamps were set
stat, err := os.Stat(testFile)
require.NoError(t, err)
require.True(t, stat.ModTime().Equal(mtime))
})
}
// TestIOOperations tests POSIX I/O operation compliance
func (s *POSIXComplianceTestSuite) TestIOOperations(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("ReadWrite", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "rw_test.txt")
testData := []byte("Hello, POSIX World!")
// Write data
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0644)
require.NoError(t, err)
n, err := syscall.Write(fd, testData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
err = syscall.Close(fd)
require.NoError(t, err)
// Read data back
fd, err = syscall.Open(testFile, syscall.O_RDONLY, 0)
require.NoError(t, err)
readBuffer := make([]byte, len(testData))
n, err = syscall.Read(fd, readBuffer)
require.NoError(t, err)
require.Equal(t, len(testData), n)
require.Equal(t, testData, readBuffer)
err = syscall.Close(fd)
require.NoError(t, err)
})
t.Run("Seek", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "seek_test.txt")
testData := []byte("0123456789ABCDEF")
// Create file with test data
err := os.WriteFile(testFile, testData, 0644)
require.NoError(t, err)
// Open for reading
fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Seek to position 5
pos, err := syscall.Seek(fd, 5, 0) // SEEK_SET
require.NoError(t, err)
require.Equal(t, int64(5), pos)
// Read 3 bytes
buffer := make([]byte, 3)
n, err := syscall.Read(fd, buffer)
require.NoError(t, err)
require.Equal(t, 3, n)
require.Equal(t, []byte("567"), buffer)
// Seek from current position
pos, err = syscall.Seek(fd, 2, 1) // SEEK_CUR
require.NoError(t, err)
require.Equal(t, int64(10), pos)
// Read 1 byte
buffer = make([]byte, 1)
n, err = syscall.Read(fd, buffer)
require.NoError(t, err)
require.Equal(t, 1, n)
require.Equal(t, []byte("A"), buffer)
// Test positioned I/O operations (pread/pwrite)
syscall.Close(fd)
// Open for read/write to test pwrite
fd, err = syscall.Open(testFile, syscall.O_RDWR, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Positioned write test
writeData := []byte("XYZ")
n, err = syscall.Pwrite(fd, writeData, 5) // pwrite at offset 5
require.NoError(t, err)
require.Equal(t, len(writeData), n)
// Verify file position is unchanged by pwrite
currentPos, err := syscall.Seek(fd, 0, 1) // SEEK_CUR
require.NoError(t, err)
require.Equal(t, int64(0), currentPos, "file offset should not be changed by pwrite")
// Read back with pread
readBuffer := make([]byte, len(writeData))
n, err = syscall.Pread(fd, readBuffer, 5) // pread at offset 5
require.NoError(t, err)
require.Equal(t, len(writeData), n)
require.Equal(t, writeData, readBuffer)
// Verify file position is still unchanged by pread
currentPos, err = syscall.Seek(fd, 0, 1) // SEEK_CUR
require.NoError(t, err)
require.Equal(t, int64(0), currentPos, "file offset should not be changed by pread")
})
t.Run("AppendMode", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "append_test.txt")
// Create file with initial content
err := os.WriteFile(testFile, []byte("initial"), 0644)
require.NoError(t, err)
// Open in append mode
fd, err := syscall.Open(testFile, syscall.O_WRONLY|syscall.O_APPEND, 0)
require.NoError(t, err)
// Write additional content
appendData := []byte(" appended")
n, err := syscall.Write(fd, appendData)
require.NoError(t, err)
require.Equal(t, len(appendData), n)
err = syscall.Close(fd)
require.NoError(t, err)
// Verify content
content, err := os.ReadFile(testFile)
require.NoError(t, err)
require.Equal(t, []byte("initial appended"), content)
})
}
// TestFileDescriptors tests POSIX file descriptor behavior
func (s *POSIXComplianceTestSuite) TestFileDescriptors(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("DuplicateFileDescriptors", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "dup_test.txt")
testData := []byte("duplicate test")
// Create and open file
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
// Write initial data
n, err := syscall.Write(fd, testData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
// Duplicate file descriptor
dupFd, err := syscall.Dup(fd)
require.NoError(t, err)
defer syscall.Close(dupFd)
// Both descriptors should refer to the same file
// Seek on one should affect the other
pos, err := syscall.Seek(fd, 0, 0) // SEEK_SET
require.NoError(t, err)
require.Equal(t, int64(0), pos)
// Read from duplicate descriptor should start from position 0
buffer := make([]byte, 9)
n, err = syscall.Read(dupFd, buffer)
require.NoError(t, err)
require.Equal(t, 9, n)
require.Equal(t, []byte("duplicate"), buffer)
})
t.Run("FileDescriptorFlags", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "flags_test.txt")
// Create file
err := os.WriteFile(testFile, []byte("test"), 0644)
require.NoError(t, err)
// Open with close-on-exec flag
fd, err := syscall.Open(testFile, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Verify close-on-exec flag is set
// Note: FcntlInt is not available on all platforms, this test may need platform-specific implementation
t.Skip("FcntlInt not available on this platform")
})
}
// TestAtomicOperations tests POSIX atomic operation compliance
func (s *POSIXComplianceTestSuite) TestAtomicOperations(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("AtomicRename", func(t *testing.T) {
oldFile := filepath.Join(mountPoint, "atomic_old.txt")
newFile := filepath.Join(mountPoint, "atomic_new.txt")
testData := []byte("atomic test data")
// Create source file
err := os.WriteFile(oldFile, testData, 0644)
require.NoError(t, err)
// Create existing target file
err = os.WriteFile(newFile, []byte("old content"), 0644)
require.NoError(t, err)
// Atomic rename should replace target
err = os.Rename(oldFile, newFile)
require.NoError(t, err)
// Verify source no longer exists
_, err = os.Stat(oldFile)
require.True(t, os.IsNotExist(err))
// Verify target has new content
content, err := os.ReadFile(newFile)
require.NoError(t, err)
require.Equal(t, testData, content)
})
}
// TestConcurrentAccess tests concurrent access patterns
func (s *POSIXComplianceTestSuite) TestConcurrentAccess(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("ConcurrentReads", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "concurrent_read.txt")
testData := []byte("concurrent read test data")
// Create test file
err := os.WriteFile(testFile, testData, 0644)
require.NoError(t, err)
// Launch multiple concurrent readers
const numReaders = 10
results := make(chan error, numReaders)
for i := 0; i < numReaders; i++ {
go func() {
content, err := os.ReadFile(testFile)
if err != nil {
results <- err
return
}
if string(content) != string(testData) {
results <- fmt.Errorf("content mismatch")
return
}
results <- nil
}()
}
// Check all readers succeeded
for i := 0; i < numReaders; i++ {
err := <-results
require.NoError(t, err)
}
})
t.Run("ConcurrentFileCreations", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "concurrent_write.txt")
// Launch multiple concurrent writers
const numWriters = 5
results := make(chan error, numWriters)
for i := 0; i < numWriters; i++ {
go func(id int) {
content := fmt.Sprintf("writer %d data", id)
err := os.WriteFile(fmt.Sprintf("%s_%d", testFile, id), []byte(content), 0644)
results <- err
}(i)
}
// Check all writers succeeded
for i := 0; i < numWriters; i++ {
err := <-results
require.NoError(t, err)
}
// Verify all files were created
for i := 0; i < numWriters; i++ {
fileName := fmt.Sprintf("%s_%d", testFile, i)
_, err := os.Stat(fileName)
require.NoError(t, err)
}
})
}
// TestErrorHandling tests POSIX error handling compliance
func (s *POSIXComplianceTestSuite) TestErrorHandling(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("AccessNonexistentFile", func(t *testing.T) {
nonexistentFile := filepath.Join(mountPoint, "does_not_exist.txt")
// Reading nonexistent file should return ENOENT
_, err := os.ReadFile(nonexistentFile)
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
t.Run("CreateFileInNonexistentDirectory", func(t *testing.T) {
fileInNonexistentDir := filepath.Join(mountPoint, "nonexistent_dir", "file.txt")
// Creating file in nonexistent directory should fail
err := os.WriteFile(fileInNonexistentDir, []byte("test"), 0644)
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
t.Run("InvalidFileDescriptor", func(t *testing.T) {
// Using an invalid file descriptor should return appropriate error
buffer := make([]byte, 10)
_, err := syscall.Read(999, buffer) // 999 is likely an invalid fd
require.Error(t, err)
require.Equal(t, syscall.EBADF, err)
})
t.Run("PermissionDenied", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("running as root: mode bits are not enforced")
}
testFile := filepath.Join(mountPoint, "readonly.txt")
// Create read-only file
err := os.WriteFile(testFile, []byte("readonly"), 0444)
require.NoError(t, err)
// Attempting to write to read-only file should fail
err = os.WriteFile(testFile, []byte("modified"), 0444)
require.Error(t, err)
require.True(t, os.IsPermission(err))
})
}
test/fuse_integration/posix_extended_test.go
+758
View File
@@ -0,0 +1,758 @@
//go:build !windows
package fuse
import (
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// POSIXExtendedTestSuite provides additional POSIX compliance tests
// covering extended attributes, file locking, and advanced features.
//
// NOTE: Some tests in this suite may be skipped or may have different behavior
// depending on the FUSE implementation and platform support. This is expected
// behavior for comprehensive POSIX compliance testing.
//
// See POSIX_IMPLEMENTATION_ROADMAP.md for a comprehensive plan to implement
// these missing features.
type POSIXExtendedTestSuite struct {
framework *FuseTestFramework
t *testing.T
}
// ErrorCollector helps collect multiple test errors without stopping execution
type ErrorCollector struct {
errors []string
t *testing.T
}
// NewErrorCollector creates a new error collector
func NewErrorCollector(t *testing.T) *ErrorCollector {
return &ErrorCollector{
errors: make([]string, 0),
t: t,
}
}
// Add adds an error to the collection
func (ec *ErrorCollector) Add(format string, args ...interface{}) {
ec.errors = append(ec.errors, fmt.Sprintf(format, args...))
}
// Check checks a condition and adds an error if it fails
func (ec *ErrorCollector) Check(condition bool, format string, args ...interface{}) {
if !condition {
ec.Add(format, args...)
}
}
// CheckError checks if an error is nil and adds it if not
func (ec *ErrorCollector) CheckError(err error, format string, args ...interface{}) {
if err != nil {
ec.Add(format+": %v", append(args, err)...)
}
}
// HasErrors returns true if any errors were collected
func (ec *ErrorCollector) HasErrors() bool {
return len(ec.errors) > 0
}
// Report reports all collected errors to the test
func (ec *ErrorCollector) Report() {
if len(ec.errors) > 0 {
ec.t.Errorf("Test completed with %d errors:\n%s", len(ec.errors), strings.Join(ec.errors, "\n"))
}
}
// NewPOSIXExtendedTestSuite creates a new extended POSIX compliance test suite
func NewPOSIXExtendedTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXExtendedTestSuite {
return &POSIXExtendedTestSuite{
framework: framework,
t: t,
}
}
// TestPOSIXExtended runs extended POSIX compliance tests
func TestPOSIXExtended(t *testing.T) {
config := DefaultTestConfig()
config.EnableDebug = true
config.MountOptions = []string{"-allowOthers", "-nonempty"}
framework := NewFuseTestFramework(t, config)
defer framework.Cleanup()
// Setup framework with better error handling
err := framework.Setup(config)
if err != nil {
t.Logf("Framework setup failed: %v", err)
t.Skip("Skipping extended tests due to framework setup failure")
return
}
// Verify framework is working
mountPoint := framework.GetMountPoint()
if mountPoint == "" {
t.Log("Framework mount point is empty")
t.Skip("Skipping extended tests due to invalid mount point")
return
}
suite := NewPOSIXExtendedTestSuite(t, framework)
// Run extended POSIX compliance test categories
// Use a resilient approach that continues even if individual tests fail
testCategories := []struct {
name string
fn func(*testing.T)
}{
{"ExtendedAttributes", suite.TestExtendedAttributes},
{"FileLocking", suite.TestFileLocking},
{"AdvancedIO", suite.TestAdvancedIO},
{"SparseFiles", suite.TestSparseFiles},
{"LargeFiles", suite.TestLargeFiles},
{"MMap", suite.TestMemoryMapping},
{"DirectIO", suite.TestDirectIO},
{"FileSealing", suite.TestFileSealing},
{"Fallocate", suite.TestFallocate},
{"Sendfile", suite.TestSendfile},
}
// Run all test categories and collect results
var failedCategories []string
for _, category := range testCategories {
t.Run(category.name, func(subT *testing.T) {
// Capture panics and convert them to test failures
defer func() {
if r := recover(); r != nil {
subT.Errorf("Test category %s panicked: %v", category.name, r)
failedCategories = append(failedCategories, category.name)
}
}()
category.fn(subT)
if subT.Failed() {
failedCategories = append(failedCategories, category.name)
}
})
}
// Report overall results
if len(failedCategories) > 0 {
t.Logf("Extended POSIX tests completed. Failed categories: %v", failedCategories)
} else {
t.Logf("All extended POSIX test categories passed!")
}
}
// TestExtendedAttributes tests POSIX extended attribute support
func (s *POSIXExtendedTestSuite) TestExtendedAttributes(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("SetAndGetXattr", func(t *testing.T) {
if !isXattrSupported() {
t.Skip("Extended attributes not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "xattr_test.txt")
// Create test file
err := os.WriteFile(testFile, []byte("xattr test"), 0644)
if !assert.NoError(t, err, "Failed to create test file") {
return
}
// Set extended attribute
attrName := "user.test_attr"
attrValue := []byte("test_value")
err = setXattr(testFile, attrName, attrValue, 0)
assert.NoError(t, err, "Failed to set extended attribute")
// Verify attribute was set
readValue := make([]byte, 256)
size, err := getXattr(testFile, attrName, readValue)
assert.NoError(t, err, "Failed to get extended attribute")
assert.Equal(t, len(attrValue), size, "Extended attribute size mismatch")
assert.Equal(t, attrValue, readValue[:size], "Extended attribute value mismatch")
})
t.Run("ListXattrs", func(t *testing.T) {
if !isXattrSupported() {
t.Skip("Extended attributes not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "xattr_list_test.txt")
// Create test file
err := os.WriteFile(testFile, []byte("list xattr test"), 0644)
if !assert.NoError(t, err, "Failed to create test file") {
return
}
// Set multiple extended attributes
attrs := map[string][]byte{
"user.attr1": []byte("value1"),
"user.attr2": []byte("value2"),
"user.attr3": []byte("value3"),
}
for name, value := range attrs {
err = setXattr(testFile, name, value, 0)
assert.NoError(t, err, "Failed to set extended attribute %s", name)
}
// List all attributes
listBuf := make([]byte, 1024)
size, err := listXattr(testFile, listBuf)
assert.NoError(t, err, "Failed to list extended attributes")
assert.Greater(t, size, 0, "No extended attributes found")
// Parse the null-separated list and verify attributes
if size > 0 {
attrList := parseXattrList(listBuf[:size])
expectedAttrs := []string{"user.attr1", "user.attr2", "user.attr3"}
for _, expectedAttr := range expectedAttrs {
assert.Contains(t, attrList, expectedAttr, "Expected attribute %s should be in the list", expectedAttr)
}
}
})
t.Run("RemoveXattr", func(t *testing.T) {
if !isXattrSupported() {
t.Skip("Extended attributes not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "xattr_remove_test.txt")
// Create test file
err := os.WriteFile(testFile, []byte("remove xattr test"), 0644)
require.NoError(t, err)
// Set extended attribute
attrName := "user.remove_test"
attrValue := []byte("to_be_removed")
err = setXattr(testFile, attrName, attrValue, 0)
require.NoError(t, err)
// Verify attribute exists
readValue := make([]byte, 256)
size, err := getXattr(testFile, attrName, readValue)
require.NoError(t, err)
require.Equal(t, len(attrValue), size)
// Remove the attribute
err = removeXattr(testFile, attrName)
require.NoError(t, err)
// Verify attribute is gone
_, err = getXattr(testFile, attrName, readValue)
require.Error(t, err) // Should fail with ENODATA or similar
})
}
// TestFileLocking tests POSIX file locking mechanisms
// Note: File locking behavior may vary between FUSE implementations.
// Some implementations may not enforce locks between file descriptors
// from the same process, which is a known limitation.
func (s *POSIXExtendedTestSuite) TestFileLocking(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("AdvisoryLocking", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "lock_test.txt")
// Create test file
err := os.WriteFile(testFile, []byte("locking test"), 0644)
if !assert.NoError(t, err, "Failed to create test file") {
return
}
// Open file
file, err := os.OpenFile(testFile, os.O_RDWR, 0644)
if !assert.NoError(t, err, "Failed to open test file") {
return
}
defer file.Close()
// Apply exclusive lock
flock := syscall.Flock_t{
Type: syscall.F_WRLCK,
Whence: 0,
Start: 0,
Len: 0, // Lock entire file
}
err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock)
assert.NoError(t, err, "Failed to acquire exclusive lock")
// Try to lock from another file descriptor (should fail)
file2, err := os.OpenFile(testFile, os.O_RDWR, 0644)
if !assert.NoError(t, err, "Failed to open second file descriptor") {
return
}
defer file2.Close()
flock2 := syscall.Flock_t{
Type: syscall.F_WRLCK,
Whence: 0,
Start: 0,
Len: 0,
}
// Note: Some FUSE implementations may not enforce file locking between file descriptors
// from the same process. This is a known limitation in some FUSE implementations.
err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
if err == syscall.EAGAIN {
// Lock was properly enforced
t.Log("File locking properly enforced between file descriptors")
} else if err == nil {
// Lock was not enforced (common in some FUSE implementations)
t.Log("File locking not enforced between file descriptors from same process (FUSE limitation)")
} else {
// Some other error occurred
t.Logf("File locking attempt resulted in error: %v", err)
}
// Log the actual error for debugging
t.Logf("File locking test completed with result: %v", err)
// Release lock
flock.Type = syscall.F_UNLCK
err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock)
assert.NoError(t, err, "Failed to release lock")
// Now second lock should succeed
err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
assert.NoError(t, err, "Failed to acquire lock after release")
})
t.Run("SharedLocking", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "shared_lock_test.txt")
// Create test file
err := os.WriteFile(testFile, []byte("shared locking test"), 0644)
if !assert.NoError(t, err, "Failed to create test file") {
return
}
// Open file for reading
file1, err := os.Open(testFile)
if !assert.NoError(t, err, "Failed to open first file descriptor") {
return
}
defer file1.Close()
file2, err := os.Open(testFile)
if !assert.NoError(t, err, "Failed to open second file descriptor") {
return
}
defer file2.Close()
// Apply shared locks (should both succeed)
flock1 := syscall.Flock_t{
Type: syscall.F_RDLCK,
Whence: 0,
Start: 0,
Len: 0,
}
flock2 := syscall.Flock_t{
Type: syscall.F_RDLCK,
Whence: 0,
Start: 0,
Len: 0,
}
err = syscall.FcntlFlock(file1.Fd(), syscall.F_SETLK, &flock1)
assert.NoError(t, err, "Failed to acquire first shared lock")
err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
assert.NoError(t, err, "Failed to acquire second shared lock")
})
}
// TestAdvancedIO tests advanced I/O operations
// Note: Vectored I/O support may vary between FUSE implementations.
// Some implementations may not properly support readv/writev operations.
func (s *POSIXExtendedTestSuite) TestAdvancedIO(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("ReadWriteV", func(t *testing.T) {
if !isVectoredIOSupported() {
t.Skip("Vectored I/O not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "readwritev_test.txt")
// Create file
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
if !assert.NoError(t, err, "Failed to create test file") {
return
}
defer syscall.Close(fd)
// Prepare test data in multiple buffers
writeBuffers := [][]byte{
[]byte("Hello "),
[]byte("vectored "),
[]byte("I/O "),
[]byte("world!"),
}
// Write using writev
writeIOVs := makeIOVecs(writeBuffers)
t.Logf("Attempting vectored I/O write with %d buffers", len(writeIOVs))
totalWritten, err := writevFile(fd, writeIOVs)
if err != nil {
// Some FUSE implementations may not support vectored I/O properly
t.Logf("Vectored I/O write failed: %v - this may be a FUSE limitation", err)
t.Skip("Vectored I/O not properly supported by this FUSE implementation")
return
}
expectedTotal := 0
for _, buf := range writeBuffers {
expectedTotal += len(buf)
}
assert.Equal(t, expectedTotal, totalWritten, "Vectored write size mismatch")
// Seek back to beginning
_, err = syscall.Seek(fd, 0, 0)
if !assert.NoError(t, err, "Failed to seek to beginning") {
return
}
// Read using readv into multiple buffers
readBuffers := [][]byte{
make([]byte, 6), // "Hello "
make([]byte, 9), // "vectored "
make([]byte, 4), // "I/O "
make([]byte, 6), // "world!"
}
readIOVs := makeIOVecs(readBuffers)
t.Logf("Attempting vectored I/O read with %d buffers", len(readIOVs))
totalRead, err := readvFile(fd, readIOVs)
if err != nil {
t.Logf("Vectored I/O read failed: %v - this may be a FUSE limitation", err)
t.Skip("Vectored I/O not properly supported by this FUSE implementation")
return
}
assert.Equal(t, expectedTotal, totalRead, "Vectored read size mismatch")
// Verify data matches
for i, expected := range writeBuffers {
assert.Equal(t, expected, readBuffers[i], "Vectored I/O data mismatch at buffer %d", i)
}
})
}
// TestSparseFiles tests sparse file handling
func (s *POSIXExtendedTestSuite) TestSparseFiles(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("CreateSparseFile", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "sparse_test.txt")
// Open file
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
// Create a sparse file by seeking beyond end and writing
const sparseSize = 1024 * 1024 // 1MB
_, err = syscall.Seek(fd, sparseSize, 0)
require.NoError(t, err)
// Write at the end
endData := []byte("end")
n, err := syscall.Write(fd, endData)
require.NoError(t, err)
require.Equal(t, len(endData), n)
// Verify file size
stat, err := os.Stat(testFile)
require.NoError(t, err)
require.Equal(t, int64(sparseSize+len(endData)), stat.Size())
// Read from the beginning (should be zeros)
_, err = syscall.Seek(fd, 0, 0)
require.NoError(t, err)
buffer := make([]byte, 100)
n, err = syscall.Read(fd, buffer)
require.NoError(t, err)
require.Equal(t, 100, n)
// Should be all zeros
for _, b := range buffer {
require.Equal(t, byte(0), b)
}
})
}
// TestLargeFiles tests large file handling (>2GB)
func (s *POSIXExtendedTestSuite) TestLargeFiles(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("LargeFileOperations", func(t *testing.T) {
testFile := filepath.Join(mountPoint, "large_file_test.txt")
// Open file
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
// Seek to position > 2GB
const largeOffset = 3 * 1024 * 1024 * 1024 // 3GB
pos, err := syscall.Seek(fd, largeOffset, 0)
require.NoError(t, err)
require.Equal(t, int64(largeOffset), pos)
// Write at large offset
testData := []byte("large file data")
n, err := syscall.Write(fd, testData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
// Read back from large offset
_, err = syscall.Seek(fd, largeOffset, 0)
require.NoError(t, err)
readBuffer := make([]byte, len(testData))
n, err = syscall.Read(fd, readBuffer)
require.NoError(t, err)
require.Equal(t, len(testData), n)
require.Equal(t, testData, readBuffer)
})
}
// TestMemoryMapping tests memory mapping functionality
func (s *POSIXExtendedTestSuite) TestMemoryMapping(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("MmapFile", func(t *testing.T) {
if !isMmapSupported() {
t.Skip("Memory mapping not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "mmap_test.txt")
testData := make([]byte, 4096)
for i := range testData {
testData[i] = byte(i % 256)
}
// Create test file
err := os.WriteFile(testFile, testData, 0644)
require.NoError(t, err)
// Open file for reading
fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Memory map the file
mappedData, err := mmapFile(fd, 0, len(testData), PROT_READ, MAP_SHARED)
require.NoError(t, err)
defer munmapFile(mappedData)
// Verify mapped content matches original
require.Equal(t, testData, mappedData)
})
t.Run("MmapWrite", func(t *testing.T) {
if !isMmapSupported() {
t.Skip("Memory mapping not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "mmap_write_test.txt")
size := 4096
// Create empty file of specific size
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
err = syscall.Ftruncate(fd, int64(size))
require.NoError(t, err)
// Memory map the file for writing
mappedData, err := mmapFile(fd, 0, size, PROT_READ|PROT_WRITE, MAP_SHARED)
require.NoError(t, err)
defer munmapFile(mappedData)
// Write test pattern to mapped memory
testPattern := []byte("Hello, mmap world!")
copy(mappedData, testPattern)
// Sync changes to disk
err = msyncFile(mappedData, MS_SYNC)
require.NoError(t, err)
// Verify changes were written by reading file directly
readData, err := os.ReadFile(testFile)
require.NoError(t, err)
require.True(t, len(readData) >= len(testPattern))
require.Equal(t, testPattern, readData[:len(testPattern)])
})
}
// TestDirectIO tests direct I/O operations
func (s *POSIXExtendedTestSuite) TestDirectIO(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("DirectIO", func(t *testing.T) {
if !isDirectIOSupported() {
t.Skip("Direct I/O not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "directio_test.txt")
// Open file with direct I/O
fd, err := openDirectIO(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
// For direct I/O, data must be aligned to sector boundaries
// Use 4KB aligned buffer (common sector size)
blockSize := 4096
testData := make([]byte, blockSize)
for i := range testData {
testData[i] = byte(i % 256)
}
// Write data using direct I/O
n, err := syscall.Write(fd, testData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
// Seek back to beginning
_, err = syscall.Seek(fd, 0, 0)
require.NoError(t, err)
// Read data using direct I/O
readData := make([]byte, blockSize)
n, err = syscall.Read(fd, readData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
require.Equal(t, testData, readData)
})
}
// TestFileSealing tests file sealing mechanisms (Linux-specific)
func (s *POSIXExtendedTestSuite) TestFileSealing(t *testing.T) {
// This test is Linux-specific and may not be applicable to all systems
t.Skip("File sealing tests require Linux-specific features")
}
// TestFallocate tests file preallocation
func (s *POSIXExtendedTestSuite) TestFallocate(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("FallocateSpace", func(t *testing.T) {
if !isFallocateSupported() {
t.Skip("fallocate not supported on this platform")
return
}
testFile := filepath.Join(mountPoint, "fallocate_test.txt")
// Open file
fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
require.NoError(t, err)
defer syscall.Close(fd)
// Preallocate 1MB of space
allocSize := int64(1024 * 1024)
err = fallocateFile(fd, 0, 0, allocSize)
require.NoError(t, err)
// Verify file size was extended
var stat syscall.Stat_t
err = syscall.Fstat(fd, &stat)
require.NoError(t, err)
require.GreaterOrEqual(t, stat.Size, allocSize)
// Write some data and verify it works
testData := []byte("fallocate test data")
n, err := syscall.Write(fd, testData)
require.NoError(t, err)
require.Equal(t, len(testData), n)
})
}
// TestSendfile tests zero-copy file transfer
func (s *POSIXExtendedTestSuite) TestSendfile(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("SendfileCopy", func(t *testing.T) {
sourceFile := filepath.Join(mountPoint, "sendfile_source.txt")
targetFile := filepath.Join(mountPoint, "sendfile_target.txt")
testData := make([]byte, 8192)
for i := range testData {
testData[i] = byte(i % 256)
}
// Create source file
err := os.WriteFile(sourceFile, testData, 0644)
require.NoError(t, err)
// Open source for reading
srcFd, err := syscall.Open(sourceFile, syscall.O_RDONLY, 0)
require.NoError(t, err)
defer syscall.Close(srcFd)
// Create target file
dstFd, err := syscall.Open(targetFile, syscall.O_CREAT|syscall.O_WRONLY, 0644)
require.NoError(t, err)
defer syscall.Close(dstFd)
// Sendfile test
if !isSendfileSupported() {
t.Skip("sendfile not supported on this platform")
return
}
// Use sendfile to copy data
var offset int64 = 0
transferred, err := sendfileTransfer(dstFd, srcFd, &offset, len(testData))
require.NoError(t, err)
require.Equal(t, len(testData), transferred)
// Verify copy
copiedData, err := os.ReadFile(targetFile)
require.NoError(t, err)
require.Equal(t, testData, copiedData)
})
}
// Helper function to parse null-separated xattr list
func parseXattrList(data []byte) []string {
var attrs []string
start := 0
for i, b := range data {
if b == 0 {
if i > start {
attrs = append(attrs, string(data[start:i]))
}
start = i + 1
}
}
return attrs
}
test/fuse_integration/posix_external_test.go
+576
View File
@@ -0,0 +1,576 @@
package fuse
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ExternalPOSIXTestSuite manages integration with external POSIX test suites
type ExternalPOSIXTestSuite struct {
framework *FuseTestFramework
t *testing.T
workDir string
}
// NewExternalPOSIXTestSuite creates a new external POSIX test suite runner
func NewExternalPOSIXTestSuite(t *testing.T, framework *FuseTestFramework) *ExternalPOSIXTestSuite {
workDir, err := os.MkdirTemp("", "posix_external_tests_")
require.NoError(t, err)
return &ExternalPOSIXTestSuite{
framework: framework,
t: t,
workDir: workDir,
}
}
// Cleanup removes temporary test directories
func (s *ExternalPOSIXTestSuite) Cleanup() {
os.RemoveAll(s.workDir)
}
// TestExternalPOSIXSuites runs integration tests with external POSIX test suites
func TestExternalPOSIXSuites(t *testing.T) {
config := DefaultTestConfig()
config.EnableDebug = true
config.MountOptions = []string{"-allowOthers", "-nonempty"}
framework := NewFuseTestFramework(t, config)
defer framework.Cleanup()
require.NoError(t, framework.Setup(config))
suite := NewExternalPOSIXTestSuite(t, framework)
defer suite.Cleanup()
// Run various external POSIX test suites
t.Run("PjdFsTest", suite.TestPjdFsTest)
t.Run("NFSTestPOSIX", suite.TestNFSTestPOSIX)
t.Run("LitmusTests", suite.TestLitmusTests)
t.Run("CustomPOSIXTests", suite.TestCustomPOSIXTests)
}
// TestPjdFsTest runs the comprehensive pjdfstest POSIX filesystem test suite
func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
// Create test directory within mount point
testDir := filepath.Join(mountPoint, "pjdfstest")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
// List of critical POSIX operations to test
pjdTests := []struct {
name string
testPath string
critical bool
}{
{"chflags", "tests/chflags", false},
{"chmod", "tests/chmod", true},
{"chown", "tests/chown", true},
{"create", "tests/create", true},
{"link", "tests/link", true},
{"mkdir", "tests/mkdir", true},
{"mkfifo", "tests/mkfifo", false},
{"mknod", "tests/mknod", false},
{"open", "tests/open", true},
{"rename", "tests/rename", true},
{"rmdir", "tests/rmdir", true},
{"symlink", "tests/symlink", true},
{"truncate", "tests/truncate", true},
{"unlink", "tests/unlink", true},
}
// Prefer the suite that `make setup-pjdfstest` builds under
// external_tools/ before cloning a second copy into the work dir
pjdDir := s.findPjdDir()
if pjdDir == "" {
pjdDir = filepath.Join(s.workDir, "pjdfstest")
if err := s.setupPjdFsTest(pjdDir); err != nil {
if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" {
t.Fatalf("pjdfstest is required for comprehensive POSIX compliance testing: %v", err)
}
t.Skipf("pjdfstest not found and setup failed: %v", err)
}
}
// Run each test category
for _, test := range pjdTests {
t.Run(test.name, func(t *testing.T) {
s.runPjdTest(t, pjdDir, test.testPath, testDir, test.critical)
})
}
}
// TestNFSTestPOSIX runs nfstest_posix for POSIX API verification
func (s *ExternalPOSIXTestSuite) TestNFSTestPOSIX(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
// Check if nfstest_posix is available
_, err := exec.LookPath("nfstest_posix")
if err != nil {
// For comprehensive POSIX compliance testing, external tools should be mandatory
if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" {
t.Fatalf("nfstest_posix is required for comprehensive POSIX compliance testing but not found. Install via: pip install nfstest")
}
t.Skip("nfstest_posix not found. Install via: pip install nfstest")
}
testDir := filepath.Join(mountPoint, "nfstest")
err = os.MkdirAll(testDir, 0755)
require.NoError(t, err)
// Run nfstest_posix with comprehensive API testing
cmd := exec.Command("nfstest_posix",
"--path", testDir,
"--verbose",
"--createlog",
"--runid", fmt.Sprintf("seaweedfs_%d", time.Now().Unix()),
)
output, err := cmd.CombinedOutput()
t.Logf("nfstest_posix output:\n%s", string(output))
if err != nil {
t.Logf("nfstest_posix failed: %v", err)
// Don't fail the test completely, just log the failure
}
}
// TestLitmusTests runs focused POSIX compliance litmus tests
func (s *ExternalPOSIXTestSuite) TestLitmusTests(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
// Create litmus test scripts for critical POSIX behaviors
litmusTests := []struct {
name string
script string
}{
{
name: "AtomicRename",
script: `#!/bin/bash
set -e
cd "$1"
echo "test data" > temp_file
echo "original" > target_file
mv temp_file target_file
[ "$(cat target_file)" = "test data" ]
echo "PASS: Atomic rename works"
`,
},
{
name: "LinkCount",
script: `#!/bin/bash
set -e
cd "$1"
echo "test" > original
ln original hardlink
if [[ "$(uname)" == "Darwin" ]]; then
[ $(stat -f %l original) -eq 2 ]
else
[ $(stat -c %h original) -eq 2 ]
fi
rm hardlink
if [[ "$(uname)" == "Darwin" ]]; then
[ $(stat -f %l original) -eq 1 ]
else
[ $(stat -c %h original) -eq 1 ]
fi
echo "PASS: Hard link counting works"
`,
},
{
name: "SymlinkCycles",
script: `#!/bin/bash
set -e
cd "$1"
ln -s link1 link2
ln -s link2 link1
# Try to access the file, which should fail due to too many symbolic links
if cat link1 2>/dev/null; then
echo "FAIL: Symlink cycle not detected"
exit 1
fi
echo "PASS: Symlink cycle handling works"
`,
},
{
name: "ConcurrentCreate",
script: `#!/bin/bash
set -e
cd "$1"
for i in {1..10}; do
(echo "process $i" > "file_$i") &
done
wait
[ $(ls file_* | wc -l) -eq 10 ]
echo "PASS: Concurrent file creation works"
`,
},
{
name: "DirectoryConsistency",
script: `#!/bin/bash
set -e
cd "$1"
mkdir testdir
cd testdir
touch file1 file2 file3
cd ..
entries=$(ls testdir | wc -l)
[ $entries -eq 3 ]
rmdir testdir 2>/dev/null && echo "FAIL: Non-empty directory removed" && exit 1
rm testdir/*
rmdir testdir
echo "PASS: Directory consistency works"
`,
},
}
testDir := filepath.Join(mountPoint, "litmus")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
// Run each litmus test
for _, test := range litmusTests {
t.Run(test.name, func(t *testing.T) {
s.runLitmusTest(t, test.name, test.script, testDir)
})
}
}
// TestCustomPOSIXTests runs custom POSIX compliance tests
func (s *ExternalPOSIXTestSuite) TestCustomPOSIXTests(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
// Custom stress tests for POSIX compliance
t.Run("StressRename", func(t *testing.T) {
s.stressTestRename(t, mountPoint)
})
t.Run("StressCreate", func(t *testing.T) {
s.stressTestCreate(t, mountPoint)
})
t.Run("StressDirectory", func(t *testing.T) {
s.stressTestDirectory(t, mountPoint)
})
t.Run("EdgeCases", func(t *testing.T) {
s.testEdgeCases(t, mountPoint)
})
}
// findPjdDir returns a directory holding a built pjdfstest suite, or ""
// when none is installed. The path is absolute because tests run with
// cmd.Dir inside the mount.
func (s *ExternalPOSIXTestSuite) findPjdDir() string {
dir := filepath.Join("external_tools", "pjdfstest")
if info, err := os.Stat(filepath.Join(dir, "pjdfstest")); err == nil && !info.IsDir() && info.Mode()&0111 != 0 {
if abs, err := filepath.Abs(dir); err == nil {
return abs
}
}
return ""
}
// setupPjdFsTest downloads and sets up the pjdfstest suite
func (s *ExternalPOSIXTestSuite) setupPjdFsTest(targetDir string) error {
// Clone pjdfstest repository
cmd := exec.Command("git", "clone", "https://github.com/pjd/pjdfstest.git", targetDir)
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to clone pjdfstest: %w", err)
}
// pjdfstest is autotools-based: the fresh clone has no Makefile until
// configure runs
for _, step := range [][]string{
{"autoreconf", "-ifs"},
{"./configure"},
{"make"},
} {
cmd = exec.Command(step[0], step[1:]...)
cmd.Dir = targetDir
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("pjdfstest %s: %w: %s", step[0], err, out)
}
}
return nil
}
// runPjdTest executes a specific pjdfstest
func (s *ExternalPOSIXTestSuite) runPjdTest(t *testing.T, pjdDir, testPath, mountDir string, critical bool) {
fullTestPath := filepath.Join(pjdDir, testPath)
// Check if test path exists
if _, err := os.Stat(fullTestPath); os.IsNotExist(err) {
t.Skipf("Test path does not exist: %s", fullTestPath)
}
cmd := exec.Command("prove", "-r", fullTestPath)
cmd.Dir = mountDir
cmd.Env = append(os.Environ(), "FSTEST="+filepath.Join(pjdDir, "pjdfstest"))
output, err := cmd.CombinedOutput()
t.Logf("pjdfstest %s output:\n%s", testPath, string(output))
if err != nil {
if critical {
t.Errorf("Critical pjdfstest failed: %s - %v", testPath, err)
} else {
t.Logf("Non-critical pjdfstest failed: %s - %v", testPath, err)
}
}
}
// runLitmusTest executes a litmus test script
func (s *ExternalPOSIXTestSuite) runLitmusTest(t *testing.T, name, script, testDir string) {
scriptPath := filepath.Join(s.workDir, name+".sh")
// Write script to file
err := os.WriteFile(scriptPath, []byte(script), 0755)
require.NoError(t, err)
// Create isolated test directory
isolatedDir := filepath.Join(testDir, name)
err = os.MkdirAll(isolatedDir, 0755)
require.NoError(t, err)
defer os.RemoveAll(isolatedDir)
// Execute script
cmd := exec.Command("bash", scriptPath, isolatedDir)
output, err := cmd.CombinedOutput()
t.Logf("Litmus test %s output:\n%s", name, string(output))
if err != nil {
t.Errorf("Litmus test %s failed: %v", name, err)
}
}
// stressTestRename tests rename operations under stress
func (s *ExternalPOSIXTestSuite) stressTestRename(t *testing.T, mountPoint string) {
testDir := filepath.Join(mountPoint, "stress_rename")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
defer os.RemoveAll(testDir)
// Create files and rename them concurrently
const numFiles = 100
const numWorkers = 10
// Create initial files
for i := 0; i < numFiles; i++ {
fileName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i))
err := os.WriteFile(fileName, []byte(fmt.Sprintf("content_%d", i)), 0644)
if !assert.NoError(t, err, "Failed to create initial file %d", i) {
return // Skip test if setup fails
}
}
// Concurrent rename operations
results := make(chan error, numWorkers)
for w := 0; w < numWorkers; w++ {
go func(worker int) {
for i := worker; i < numFiles; i += numWorkers {
oldName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i))
newName := filepath.Join(testDir, fmt.Sprintf("renamed_%d.txt", i))
err := os.Rename(oldName, newName)
if err != nil {
results <- err
return
}
}
results <- nil
}(w)
}
// Wait for all workers and collect errors
var errorCount int
for w := 0; w < numWorkers; w++ {
err := <-results
if err != nil {
assert.NoError(t, err, "Worker %d failed", w)
errorCount++
}
}
// Verify all files were renamed (if no errors occurred)
if errorCount == 0 {
files, err := filepath.Glob(filepath.Join(testDir, "renamed_*.txt"))
assert.NoError(t, err, "Failed to list renamed files")
assert.Equal(t, numFiles, len(files), "Not all files were renamed successfully")
}
}
// stressTestCreate tests file creation under stress
func (s *ExternalPOSIXTestSuite) stressTestCreate(t *testing.T, mountPoint string) {
testDir := filepath.Join(mountPoint, "stress_create")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
defer os.RemoveAll(testDir)
const numFiles = 1000
const numWorkers = 20
results := make(chan error, numWorkers)
for w := 0; w < numWorkers; w++ {
go func(worker int) {
for i := worker; i < numFiles; i += numWorkers {
fileName := filepath.Join(testDir, fmt.Sprintf("stress_%d_%d.txt", worker, i))
err := os.WriteFile(fileName, []byte(fmt.Sprintf("data_%d_%d", worker, i)), 0644)
if err != nil {
results <- err
return
}
}
results <- nil
}(w)
}
// Wait for all workers
for w := 0; w < numWorkers; w++ {
err := <-results
require.NoError(t, err)
}
// Verify all files were created
files, err := filepath.Glob(filepath.Join(testDir, "stress_*.txt"))
require.NoError(t, err)
require.Equal(t, numFiles, len(files))
}
// stressTestDirectory tests directory operations under stress
func (s *ExternalPOSIXTestSuite) stressTestDirectory(t *testing.T, mountPoint string) {
testDir := filepath.Join(mountPoint, "stress_directory")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
defer os.RemoveAll(testDir)
const numDirs = 500
const numWorkers = 10
results := make(chan error, numWorkers)
// Create directories concurrently
for w := 0; w < numWorkers; w++ {
go func(worker int) {
for i := worker; i < numDirs; i += numWorkers {
dirName := filepath.Join(testDir, fmt.Sprintf("dir_%d_%d", worker, i))
err := os.MkdirAll(dirName, 0755)
if err != nil {
results <- err
return
}
// Create a file in each directory
fileName := filepath.Join(dirName, "test.txt")
err = os.WriteFile(fileName, []byte("test"), 0644)
if err != nil {
results <- err
return
}
}
results <- nil
}(w)
}
// Wait for all workers
for w := 0; w < numWorkers; w++ {
err := <-results
require.NoError(t, err)
}
// Verify directory structure
dirs, err := filepath.Glob(filepath.Join(testDir, "dir_*"))
require.NoError(t, err)
require.Equal(t, numDirs, len(dirs))
}
// testEdgeCases tests various POSIX edge cases
func (s *ExternalPOSIXTestSuite) testEdgeCases(t *testing.T, mountPoint string) {
testDir := filepath.Join(mountPoint, "edge_cases")
err := os.MkdirAll(testDir, 0755)
require.NoError(t, err)
defer os.RemoveAll(testDir)
t.Run("WriteToDirectoryAsFile", func(t *testing.T) {
// Test writing to a directory as if it were a file (should fail)
// Note: filepath.Join(testDir, "") returns testDir itself
err := os.WriteFile(testDir, []byte("test"), 0644)
require.Error(t, err)
// Verify the error is specifically about the target being a directory
var pathErr *os.PathError
require.ErrorAs(t, err, &pathErr)
require.Equal(t, syscall.EISDIR, pathErr.Err)
})
t.Run("VeryLongFileName", func(t *testing.T) {
// Test very long file names
longName := strings.Repeat("a", 255) // NAME_MAX is typically 255
longFile := filepath.Join(testDir, longName)
err := os.WriteFile(longFile, []byte("long name test"), 0644)
// This might succeed or fail depending on filesystem limits
if err == nil {
defer os.Remove(longFile)
t.Logf("Very long filename accepted")
} else {
t.Logf("Very long filename rejected: %v", err)
}
})
t.Run("SpecialCharacters", func(t *testing.T) {
// Test files with special characters
specialFiles := []string{
"file with spaces.txt",
"file-with-dashes.txt",
"file_with_underscores.txt",
"file.with.dots.txt",
}
for _, fileName := range specialFiles {
filePath := filepath.Join(testDir, fileName)
err := os.WriteFile(filePath, []byte("special char test"), 0644)
require.NoError(t, err, "Failed to create file with special chars: %s", fileName)
// Verify we can read it back
_, err = os.ReadFile(filePath)
require.NoError(t, err, "Failed to read file with special chars: %s", fileName)
}
})
t.Run("DeepDirectoryNesting", func(t *testing.T) {
// Test deep directory nesting
deepPath := testDir
for i := 0; i < 100; i++ {
deepPath = filepath.Join(deepPath, fmt.Sprintf("level_%d", i))
}
err := os.MkdirAll(deepPath, 0755)
// This might fail due to PATH_MAX limits
if err != nil {
t.Logf("Deep directory nesting failed at some level: %v", err)
} else {
t.Logf("Deep directory nesting succeeded")
// Test creating a file in the deep path
testFile := filepath.Join(deepPath, "deep_file.txt")
err = os.WriteFile(testFile, []byte("deep test"), 0644)
if err == nil {
t.Logf("File creation in deep path succeeded")
} else {
t.Logf("File creation in deep path failed: %v", err)
}
}
})
}
test/fuse_integration/sendfile_darwin.go
+45
View File
@@ -0,0 +1,45 @@
//go:build darwin
package fuse
import (
"syscall"
"unsafe"
)
// Sendfile support for macOS
func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
// macOS sendfile has different signature: sendfile(in_fd, out_fd, offset, len, hdtr, flags)
var off int64
if offset != nil {
off = *offset
}
length := int64(count)
_, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE,
uintptr(inFd), // input fd
uintptr(outFd), // output fd
uintptr(off), // offset
uintptr(unsafe.Pointer(&length)), // length (in/out parameter)
0, // hdtr (headers/trailers)
0) // flags
if errno != 0 {
return 0, errno
}
// Update offset if provided
if offset != nil {
*offset += length
}
return int(length), nil
}
func isSendfileSupported() bool {
// macOS sendfile(2) requires a socket destination; file-to-file copies
// fail with ENOTSOCK.
return false
}
test/fuse_integration/sendfile_linux.go
+33
View File
@@ -0,0 +1,33 @@
//go:build linux
package fuse
import (
"syscall"
"unsafe"
)
// Sendfile support for Linux
func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
var offsetPtr uintptr
if offset != nil {
offsetPtr = uintptr(unsafe.Pointer(offset))
}
n, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE,
uintptr(outFd),
uintptr(inFd),
offsetPtr,
uintptr(count),
0, 0)
if errno != 0 {
return 0, errno
}
return int(n), nil
}
func isSendfileSupported() bool {
return true
}
test/fuse_integration/sendfile_unsupported.go
+19
View File
@@ -0,0 +1,19 @@
//go:build !linux && !darwin
package fuse
import (
"errors"
)
// Sendfile support for unsupported platforms
var ErrSendfileNotSupported = errors.New("sendfile not supported on this platform")
func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
return 0, ErrSendfileNotSupported
}
func isSendfileSupported() bool {
return false
}
test/fuse_integration/vectored_io_unix.go
+126
View File
@@ -0,0 +1,126 @@
//go:build unix
package fuse
import (
"syscall"
"unsafe"
)
// Vectored I/O support for Unix-like systems
// IOVec represents an I/O vector for readv/writev operations
type IOVec struct {
Base *byte
Len uint64
}
func readvFile(fd int, iovs []IOVec) (int, error) {
if len(iovs) == 0 {
return 0, nil
}
n, _, errno := syscall.Syscall(syscall.SYS_READV,
uintptr(fd),
uintptr(unsafe.Pointer(&iovs[0])),
uintptr(len(iovs)))
if errno != 0 {
return 0, errno
}
return int(n), nil
}
func writevFile(fd int, iovs []IOVec) (int, error) {
if len(iovs) == 0 {
return 0, nil
}
n, _, errno := syscall.Syscall(syscall.SYS_WRITEV,
uintptr(fd),
uintptr(unsafe.Pointer(&iovs[0])),
uintptr(len(iovs)))
if errno != 0 {
return 0, errno
}
return int(n), nil
}
func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) {
if len(iovs) == 0 {
return 0, nil
}
// preadv/pwritev may not be available on all Unix systems
// Fall back to individual pread calls
totalRead := 0
currentOffset := offset
for _, iov := range iovs {
if iov.Len == 0 {
continue
}
buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len]
n, err := syscall.Pread(fd, buf, currentOffset)
if err != nil {
return totalRead, err
}
totalRead += n
currentOffset += int64(n)
if n < int(iov.Len) {
break // EOF or partial read
}
}
return totalRead, nil
}
func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) {
if len(iovs) == 0 {
return 0, nil
}
// preadv/pwritev may not be available on all Unix systems
// Fall back to individual pwrite calls
totalWritten := 0
currentOffset := offset
for _, iov := range iovs {
if iov.Len == 0 {
continue
}
buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len]
n, err := syscall.Pwrite(fd, buf, currentOffset)
if err != nil {
return totalWritten, err
}
totalWritten += n
currentOffset += int64(n)
if n < int(iov.Len) {
break // Partial write
}
}
return totalWritten, nil
}
// Helper function to create IOVec from byte slices
func makeIOVecs(buffers [][]byte) []IOVec {
iovs := make([]IOVec, len(buffers))
for i, buf := range buffers {
if len(buf) > 0 {
iovs[i].Base = &buf[0]
iovs[i].Len = uint64(len(buf))
}
}
return iovs
}
func isVectoredIOSupported() bool {
return true
}
test/fuse_integration/vectored_io_unsupported.go
+41
View File
@@ -0,0 +1,41 @@
//go:build !unix
package fuse
import (
"errors"
)
// Vectored I/O support for unsupported platforms
var ErrVectoredIONotSupported = errors.New("vectored I/O not supported on this platform")
// IOVec represents an I/O vector for readv/writev operations
type IOVec struct {
Base *byte
Len uint64
}
func readvFile(fd int, iovs []IOVec) (int, error) {
return 0, ErrVectoredIONotSupported
}
func writevFile(fd int, iovs []IOVec) (int, error) {
return 0, ErrVectoredIONotSupported
}
func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) {
return 0, ErrVectoredIONotSupported
}
func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) {
return 0, ErrVectoredIONotSupported
}
func makeIOVecs(buffers [][]byte) []IOVec {
return nil
}
func isVectoredIOSupported() bool {
return false
}
test/fuse_integration/xattr_darwin.go
+125
View File
@@ -0,0 +1,125 @@
//go:build darwin
package fuse
import (
"syscall"
"unsafe"
)
// Extended attributes support for macOS
const (
// macOS-specific flags
XATTR_NOFOLLOW = 0x0001
XATTR_CREATE = 0x0002
XATTR_REPLACE = 0x0004
)
func setXattr(path, name string, value []byte, flags int) error {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return err
}
var valuePtr unsafe.Pointer
if len(value) > 0 {
valuePtr = unsafe.Pointer(&value[0])
}
_, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
uintptr(valuePtr),
uintptr(len(value)),
uintptr(0), // position (not used for regular files)
uintptr(flags))
if errno != 0 {
return errno
}
return nil
}
func getXattr(path, name string, value []byte) (int, error) {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return 0, err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return 0, err
}
var valuePtr unsafe.Pointer
if len(value) > 0 {
valuePtr = unsafe.Pointer(&value[0])
}
size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
uintptr(valuePtr),
uintptr(len(value)),
uintptr(0), // position
uintptr(0)) // options
if errno != 0 {
return 0, errno
}
return int(size), nil
}
func listXattr(path string, list []byte) (int, error) {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return 0, err
}
var listPtr unsafe.Pointer
if len(list) > 0 {
listPtr = unsafe.Pointer(&list[0])
}
size, _, errno := syscall.Syscall6(syscall.SYS_LISTXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(listPtr),
uintptr(len(list)),
uintptr(0), // options
0, 0)
if errno != 0 {
return 0, errno
}
return int(size), nil
}
func removeXattr(path, name string) error {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return err
}
_, _, errno := syscall.Syscall6(syscall.SYS_REMOVEXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
uintptr(0), // options
0, 0, 0)
if errno != 0 {
return errno
}
return nil
}
func isXattrSupported() bool {
return true
}
test/fuse_integration/xattr_linux.go
+115
View File
@@ -0,0 +1,115 @@
//go:build linux
package fuse
import (
"syscall"
"unsafe"
)
// Extended attributes support for Linux
func setXattr(path, name string, value []byte, flags int) error {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return err
}
var valuePtr unsafe.Pointer
if len(value) > 0 {
valuePtr = unsafe.Pointer(&value[0])
}
_, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
uintptr(valuePtr),
uintptr(len(value)),
uintptr(flags),
0)
if errno != 0 {
return errno
}
return nil
}
func getXattr(path, name string, value []byte) (int, error) {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return 0, err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return 0, err
}
var valuePtr unsafe.Pointer
if len(value) > 0 {
valuePtr = unsafe.Pointer(&value[0])
}
size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
uintptr(valuePtr),
uintptr(len(value)),
0,
0)
if errno != 0 {
return 0, errno
}
return int(size), nil
}
func listXattr(path string, list []byte) (int, error) {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return 0, err
}
var listPtr unsafe.Pointer
if len(list) > 0 {
listPtr = unsafe.Pointer(&list[0])
}
size, _, errno := syscall.Syscall(syscall.SYS_LISTXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(listPtr),
uintptr(len(list)))
if errno != 0 {
return 0, errno
}
return int(size), nil
}
func removeXattr(path, name string) error {
pathBytes, err := syscall.BytePtrFromString(path)
if err != nil {
return err
}
nameBytes, err := syscall.BytePtrFromString(name)
if err != nil {
return err
}
_, _, errno := syscall.Syscall(syscall.SYS_REMOVEXATTR,
uintptr(unsafe.Pointer(pathBytes)),
uintptr(unsafe.Pointer(nameBytes)),
0)
if errno != 0 {
return errno
}
return nil
}
func isXattrSupported() bool {
return true
}
test/fuse_integration/xattr_unsupported.go
+31
View File
@@ -0,0 +1,31 @@
//go:build !linux && !darwin
package fuse
import (
"errors"
)
// Extended attributes support for unsupported platforms
var ErrXattrNotSupported = errors.New("extended attributes not supported on this platform")
func setXattr(path, name string, value []byte, flags int) error {
return ErrXattrNotSupported
}
func getXattr(path, name string, value []byte) (int, error) {
return 0, ErrXattrNotSupported
}
func listXattr(path string, list []byte) (int, error) {
return 0, ErrXattrNotSupported
}
func removeXattr(path, name string) error {
return ErrXattrNotSupported
}
func isXattrSupported() bool {
return false
}
test/kafka/simple-publisher/go.mod
+2 -2
View File
@@ -1,10 +1,10 @@
module simple-publisher
go 1.21
go 1.25
require github.com/segmentio/kafka-go v0.4.47
require (
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/compress v1.20.3 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
)
test/kafka/simple-publisher/go.sum
+2 -2
View File
@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
test/s3/versioning/s3_listing_regression_test.go
+112
View File
@@ -0,0 +1,112 @@
package s3api
import (
"bytes"
"context"
"fmt"
"sort"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Check both wire APIs with small pages. The directory walker alone does
// not exercise prefix normalization, delimiter-marker adjustment, or tokens.
func TestListingPrefixAndDelimiterPagination(t *testing.T) {
client := getS3Client(t)
ctx := context.Background()
for _, versioning := range []string{"", "Enabled", "Suspended"} {
t.Run("versioning="+versioning, func(t *testing.T) {
bucket := getNewBucketName()
createBucket(t, client, bucket)
defer deleteBucket(t, client, bucket)
if versioning != "" {
enableVersioning(t, client, bucket)
if versioning == "Suspended" {
_, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: aws.String(bucket), VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended},
})
require.NoError(t, err)
}
}
for _, key := range []string{"a", "a-c", "a.txt", "a/b", "foo_bar", "foo_bar_baz", "foo_baz"} {
_, err := client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader([]byte("x"))})
require.NoError(t, err)
}
for _, tc := range []struct {
name, prefix, delimiter, startAfter string
keys, prefixes []string
}{
{"trailing slash", "a/", "", "", []string{"a/b"}, nil},
{"trailing slash custom delimiter", "a/", "_", "", []string{"a/b"}, nil},
{"custom delimiter", "foo", "_", "", nil, []string{"foo_"}},
{"custom delimiter nested group", "foo_", "_", "", []string{"foo_bar", "foo_baz"}, []string{"foo_bar_"}},
{"marker names directory", "a", "", "a", []string{"a-c", "a.txt", "a/b"}, nil},
{"marker between name and slash", "a", "", "a.txt", []string{"a/b"}, nil},
} {
for _, v2 := range []bool{false, true} {
for _, pageSize := range []int32{1, 2, 1000} {
t.Run(fmt.Sprintf("%s/v2=%t/max=%d", tc.name, v2, pageSize), func(t *testing.T) {
var keys, prefixes []string
marker, token := tc.startAfter, ""
for page := 0; ; page++ {
require.Less(t, page, 10, "listing must terminate without repeating a prefix")
var objects []types.Object
var commonPrefixes []types.CommonPrefix
var truncated bool
if v2 {
in := &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(tc.prefix), Delimiter: aws.String(tc.delimiter), MaxKeys: aws.Int32(pageSize)}
if token != "" {
in.ContinuationToken = aws.String(token)
} else if marker != "" {
in.StartAfter = aws.String(marker)
}
out, err := client.ListObjectsV2(ctx, in)
require.NoError(t, err)
objects, commonPrefixes, truncated = out.Contents, out.CommonPrefixes, aws.ToBool(out.IsTruncated)
next := aws.ToString(out.NextContinuationToken)
if truncated {
require.NotEmpty(t, next)
require.NotEqual(t, token, next, "token must advance")
}
token = next
} else {
out, err := client.ListObjects(ctx, &s3.ListObjectsInput{
Bucket: aws.String(bucket), Prefix: aws.String(tc.prefix), Delimiter: aws.String(tc.delimiter), MaxKeys: aws.Int32(pageSize), Marker: aws.String(marker),
})
require.NoError(t, err)
objects, commonPrefixes, truncated = out.Contents, out.CommonPrefixes, aws.ToBool(out.IsTruncated)
next := aws.ToString(out.NextMarker)
if truncated {
if next == "" && len(objects) > 0 {
next = aws.ToString(objects[len(objects)-1].Key)
}
require.Greater(t, next, marker, "marker must advance")
}
marker = next
}
for _, object := range objects {
keys = append(keys, aws.ToString(object.Key))
}
for _, prefix := range commonPrefixes {
prefixes = append(prefixes, aws.ToString(prefix.Prefix))
}
if !truncated {
break
}
}
sort.Strings(keys)
sort.Strings(prefixes)
assert.Equal(t, tc.keys, keys)
assert.Equal(t, tc.prefixes, prefixes)
})
}
}
}
})
}
}
weed/admin/handlers/file_browser_grpc.go
+3
View File
@@ -153,6 +153,9 @@ func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath strin
ChunkSize: adminUploadChunkSize,
MimeType: mimeType,
AssignFunc: assignFunc,
LookupJwtFunc: func(lookupCtx context.Context, fileId string) security.EncodedJwt {
return operation.LookupJwt(lookupCtx, h.adminServer.GetMasterClient().GetMaster(lookupCtx), h.adminServer.GetGrpcDialOption(), fileId)
},
})
if err != nil {
// Partial chunks come back even on error so we can clean them up rather
weed/admin/plugin/plugin_scheduler.go
+3 -2
View File
@@ -12,6 +12,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -614,7 +615,7 @@ func deriveSchedulerAdminRuntime(
descriptor *plugin_pb.JobTypeDescriptor,
) *plugin_pb.AdminRuntimeConfig {
if cfg != nil && cfg.AdminRuntime != nil {
adminConfig := *cfg.AdminRuntime
adminConfig := proto.Clone(cfg.AdminRuntime).(*plugin_pb.AdminRuntimeConfig)
// Overlay descriptor defaults for any zero numeric fields. Persisted
// configs from older versions have no execution_timeout_seconds, and
// without this overlay the scheduler would fall back to the 90s
@@ -646,7 +647,7 @@ func deriveSchedulerAdminRuntime(
adminConfig.ExecutionTimeoutSeconds = defaults.ExecutionTimeoutSeconds
}
}
return &adminConfig
return adminConfig
}
if descriptor == nil || descriptor.AdminRuntimeDefaults == nil {
weed/admin/plugin/registry.go
+20 -20
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/proto"
)
const defaultWorkerStaleTimeout = 2 * time.Minute
@@ -495,36 +496,35 @@ func cloneJobTypeCapability(in *plugin_pb.JobTypeCapability) *plugin_pb.JobTypeC
if in == nil {
return nil
}
out := *in
return &out
return proto.Clone(in).(*plugin_pb.JobTypeCapability)
}
func cloneWorkerHeartbeat(in *plugin_pb.WorkerHeartbeat) *plugin_pb.WorkerHeartbeat {
if in == nil {
return nil
}
out := *in
out := proto.Clone(in).(*plugin_pb.WorkerHeartbeat)
if in.RunningWork != nil {
out.RunningWork = make([]*plugin_pb.RunningWork, 0, len(in.RunningWork))
for _, rw := range in.RunningWork {
if rw == nil {
continue
running := out.RunningWork[:0]
// Clone can turn nil messages into empty messages, so filter using
// their original positions while reusing the cloned entries.
for i, work := range in.RunningWork {
if work != nil {
running = append(running, out.RunningWork[i])
}
clone := *rw
out.RunningWork = append(out.RunningWork, &clone)
}
}
if in.QueuedJobsByType != nil {
out.QueuedJobsByType = make(map[string]int32, len(in.QueuedJobsByType))
for k, v := range in.QueuedJobsByType {
out.QueuedJobsByType[k] = v
clear(out.RunningWork[len(running):])
if running == nil {
running = []*plugin_pb.RunningWork{}
}
out.RunningWork = running
}
if in.Metadata != nil {
out.Metadata = make(map[string]string, len(in.Metadata))
for k, v := range in.Metadata {
out.Metadata[k] = v
}
// Preserve writable empty maps without rebuilding populated clones.
if in.QueuedJobsByType != nil && out.QueuedJobsByType == nil {
out.QueuedJobsByType = make(map[string]int32)
}
return &out
if in.Metadata != nil && out.Metadata == nil {
out.Metadata = make(map[string]string)
}
return out
}
weed/admin/plugin/registry_snapshot_test.go
+65
View File
@@ -0,0 +1,65 @@
package plugin
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestCloneWorkerHeartbeatFiltersNilAndIsolatesState(t *testing.T) {
t.Parallel()
source := &plugin_pb.WorkerHeartbeat{
RunningWork: []*plugin_pb.RunningWork{nil, {WorkId: "first"}, nil, {WorkId: "second"}, nil},
QueuedJobsByType: map[string]int32{"vacuum": 2},
Metadata: map[string]string{"rack": "rack-a"},
}
got := cloneWorkerHeartbeat(source)
if len(got.RunningWork) != 2 || got.RunningWork[0].GetWorkId() != "first" || got.RunningWork[1].GetWorkId() != "second" {
t.Fatalf("unexpected filtered work: %v", got.RunningWork)
}
got.RunningWork[0].WorkId = "changed"
got.QueuedJobsByType["vacuum"] = 99
got.Metadata["rack"] = "changed"
if len(source.RunningWork) != 5 || source.RunningWork[0] != nil || source.RunningWork[1].WorkId != "first" {
t.Fatalf("cloning mutated source work: %v", source.RunningWork)
}
if source.QueuedJobsByType["vacuum"] != 2 || source.Metadata["rack"] != "rack-a" {
t.Fatal("cloned maps alias the source")
}
}
func TestCloneWorkerHeartbeatPreservesEmptyCollections(t *testing.T) {
t.Parallel()
if cloneWorkerHeartbeat(nil) != nil {
t.Fatal("nil heartbeat must stay nil")
}
for _, tc := range []struct {
name string
source *plugin_pb.WorkerHeartbeat
wantNilWork bool
}{
{"nil", &plugin_pb.WorkerHeartbeat{}, true},
{"empty", &plugin_pb.WorkerHeartbeat{
RunningWork: []*plugin_pb.RunningWork{},
QueuedJobsByType: map[string]int32{},
Metadata: map[string]string{},
}, false},
{"all nil work", &plugin_pb.WorkerHeartbeat{RunningWork: []*plugin_pb.RunningWork{nil, nil}}, false},
} {
t.Run(tc.name, func(t *testing.T) {
got := cloneWorkerHeartbeat(tc.source)
if len(got.RunningWork) != 0 || (got.RunningWork == nil) != tc.wantNilWork {
t.Fatalf("unexpected empty work slice: %#v", got.RunningWork)
}
if (got.QueuedJobsByType == nil) != (tc.source.QueuedJobsByType == nil) || (got.Metadata == nil) != (tc.source.Metadata == nil) {
t.Fatal("clone changed whether empty maps are writable")
}
if got.Metadata != nil {
got.Metadata["added"] = "value"
if len(tc.source.Metadata) != 0 {
t.Fatal("empty metadata map aliases source")
}
}
})
}
}
weed/cluster/lock_ring_manager.go
+2 -3
View File
@@ -7,6 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/protobuf/proto"
)
const LockRingStabilizationInterval = 1 * time.Second
@@ -102,9 +103,7 @@ func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.
if !ok || update == nil {
return nil
}
cp := *update
cp.Servers = append([]string(nil), update.Servers...)
return &cp
return proto.Clone(update).(*master_pb.LockRingUpdate)
}
// scheduleBroadcast resets the stabilization timer. If another change arrives
weed/command/benchmark.go
+1 -1
View File
@@ -238,7 +238,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
}
var jwtAuthorization security.EncodedJwt
if isSecure {
jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid)
jwtAuthorization = operation.LookupJwt(context.Background(), b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid)
}
if e := util_http.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
s.completed++
weed/command/filer.go
+12
View File
@@ -51,6 +51,15 @@ var (
// S3 gateway, whose remote-mount read paths dial the mounted endpoint directly.
const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only entry accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints."
// iamSelfServiceRequiresGrantUsage documents the flag shared by every command
// that runs the embedded IAM API.
const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity."
// allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal
// object-key character instead of normalizing it to '/'. Repeated '//' in a
// key still collapses: a filer entry name cannot contain an empty segment.
const allowBackslashInKeyUsage = "treat backslash as a regular object-key character instead of converting it to '/' (aws-compatible; '//' still collapses)"
type FilerOptions struct {
masters *pb.ServerDiscovery
mastersString *string
@@ -173,6 +182,8 @@ func init() {
filerS3Options.cacheSizeMB = cmdFiler.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
filerS3Options.readerCacheSizeMB = cmdFiler.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
// start webdav on filer
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -191,6 +202,7 @@ func init() {
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
filerIamOptions.selfServiceRequiresGrant = filerS3Options.iamSelfServiceRequiresGrant
filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server")
filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port")
weed/command/filer_remote_gateway_buckets.go
+18
View File
@@ -214,11 +214,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
}
if filer_pb.IsCreate(resp) {
if message.NewParentPath == option.bucketsDir {
if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
return nil
}
return handleCreateBucket(message.NewEntry)
}
if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
return nil
}
if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
return nil
}
// Propagate delete markers as deletions on the remote.
// Delete markers are zero-content version entries, so they
// would be filtered out by the HasData check below.
@@ -292,6 +298,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
return nil
}
if resp.Directory == option.bucketsDir {
if isSystemPath(resp.Directory, message.OldEntry.Name) {
return nil
}
return handleDeleteBucket(message.OldEntry)
}
// Skip deletion of internal version files; individual version
@@ -300,6 +309,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
return nil
}
if isSystemPath(resp.Directory, message.OldEntry.Name) {
return nil
}
bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
if !ok {
return nil
@@ -323,6 +335,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
if message.OldEntry.Name == message.NewEntry.Name {
return nil
}
if isSystemPath(resp.Directory, message.OldEntry.Name) || isSystemPath(message.NewParentPath, message.NewEntry.Name) {
return nil
}
if err := handleCreateBucket(message.NewEntry); err != nil {
return err
}
@@ -339,6 +354,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
return nil
}
if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
return nil
}
oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
if oldOk && newOk {
weed/command/filer_remote_sync_dir.go
+30 -1
View File
@@ -144,6 +144,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
return nil
}
if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
return nil
}
// Propagate delete markers as deletions on the remote.
// Delete markers are zero-content version entries, so they
// would be filtered out by the HasData check below.
@@ -229,6 +233,10 @@ func processDeleteEvent(
glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
return nil
}
if isSystemPath(resp.Directory, message.OldEntry.Name) {
glog.V(2).Infof("skipping delete of system path: %s/%s", resp.Directory, message.OldEntry.Name)
return nil
}
if filer.IsMetadataOnlyDelete(message.OldEntry) {
glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
return nil
@@ -339,6 +347,10 @@ func processUpdateEvent(
glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
return nil
}
if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
glog.V(2).Infof("skipping update of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
return nil
}
oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()}
@@ -428,7 +440,7 @@ func processUpdateEvent(
}
glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
if err := client.DeleteFile(oldDest); err != nil {
if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) {
if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) || isSystemPath(resp.Directory, message.OldEntry.Name) {
return nil
}
if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
@@ -881,6 +893,23 @@ func isMultipartUploadDir(dir string) bool {
strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/")
}
// isSystemPath returns true if the dir and name refer to the internal .system folder
// under /buckets (such as the bucket-owner index at /buckets/.system/owners/...)
// or any entry inside it. These paths are internal to SeaweedFS and must not
// be synced to remote storage.
func isSystemPath(dir string, name string) bool {
fullPath := string(util.NewFullPath(dir, name))
trimmed := strings.Trim(fullPath, "/")
parts := strings.Split(trimmed, "/")
if len(parts) >= 1 && parts[0] == s3_constants.SystemFolder {
return true
}
if len(parts) >= 2 && parts[1] == s3_constants.SystemFolder {
return true
}
return false
}
// isDeleteMarker returns true if the entry is an S3 delete marker
// (a zero-content version entry with ExtDeleteMarkerKey set to "true").
func isDeleteMarker(entry *filer_pb.Entry) bool {
weed/command/filer_remote_sync_dir_test.go
+71
View File
@@ -2016,3 +2016,74 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) {
t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry)
}
}
func TestIsSystemPath(t *testing.T) {
tests := []struct {
dir string
name string
expected bool
}{
{dir: "/buckets", name: ".system", expected: true},
{dir: "/buckets/.system", name: "owners", expected: true},
{dir: "/buckets/.system/owners", name: "alice", expected: true},
{dir: "/buckets/.system/owners/alice", name: "mybucket", expected: true},
{dir: "/", name: ".system", expected: true},
{dir: "/.system", name: "config", expected: true},
{dir: "/custom-mount", name: ".system", expected: true},
{dir: "/custom-mount/.system/owners", name: "alice", expected: true},
// Bucket contents should not match
{dir: "/buckets/mybucket", name: "file.txt", expected: false},
{dir: "/buckets/mybucket", name: ".system", expected: false},
{dir: "/buckets/mybucket/.system", name: "data", expected: false},
{dir: "/buckets", name: ".system-other", expected: false},
{dir: "/buckets/.system-other", name: "sub", expected: false},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("%s/%s", tt.dir, tt.name), func(t *testing.T) {
got := isSystemPath(tt.dir, tt.name)
if got != tt.expected {
t.Errorf("isSystemPath(%q, %q) = %v, want %v", tt.dir, tt.name, got, tt.expected)
}
})
}
}
func TestRemoteSyncSkipsSystemFolder(t *testing.T) {
const mountedDir = "/buckets"
mountLoc := &remote_pb.RemoteStorageLocation{Name: "azure", Bucket: "project-storage", Path: "/"}
// Test update event on /buckets/.system/owners/... is skipped
updateResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/buckets/.system/owners/structify-api",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"},
NewParentPath: "/buckets/.system/owners/structify-api",
NewEntry: &filer_pb.Entry{Name: "structify-copy-canary"},
},
}
remote := &recordingRemote{}
filerClient := &stubFilerClient{}
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, updateResp); err != nil {
t.Fatalf("processUpdateEvent failed: %v", err)
}
if len(remote.writes) != 0 || len(remote.metadataUpdates) != 0 {
t.Fatalf("expected 0 remote writes or metadata updates for system path, got %d writes and %d metadata updates",
len(remote.writes), len(remote.metadataUpdates))
}
// Test delete event on /buckets/.system/owners/... is skipped
deleteResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/buckets/.system/owners/structify-api",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"},
},
}
if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleteResp); err != nil {
t.Fatalf("processDeleteEvent failed: %v", err)
}
if len(remote.deletes) != 0 {
t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes))
}
}
weed/command/iam.go
+11 -8
View File
@@ -28,10 +28,11 @@ var (
)
type IamOptions struct {
filer *string
masters *string
ip *string
port *int
filer *string
masters *string
ip *string
port *int
selfServiceRequiresGrant *bool
}
func init() {
@@ -40,6 +41,7 @@ func init() {
iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
iamStandaloneOptions.selfServiceRequiresGrant = cmdIam.Flag.Bool("selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
}
var cmdIam = &Command{
@@ -101,10 +103,11 @@ func (iamopt *IamOptions) startIamServer() bool {
masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
router := mux.NewRouter().SkipClean(true)
iamApiServer, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
Masters: masters,
Filers: filerAddresses,
Port: *iamopt.port,
GrpcDialOption: grpcDialOption,
Masters: masters,
Filers: filerAddresses,
Port: *iamopt.port,
GrpcDialOption: grpcDialOption,
IamSelfServiceRequiresGrant: *iamopt.selfServiceRequiresGrant,
})
glog.V(0).Info("NewIamApiServer created")
if iamApiServer_err != nil {
weed/command/mini.go
+2
View File
@@ -556,6 +556,8 @@ func initMiniS3Flags() {
miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
miniS3Options.readerCacheSizeMB = cmdMini.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
miniS3Options.debug = new(bool) // explicitly false
miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")
weed/command/s3.go
+8 -1
View File
@@ -84,6 +84,8 @@ type S3Options struct {
readerCacheSizeMB *int64
allowUntrustedRemoteEndpoints *bool
allowBackslashInKey *bool
iamSelfServiceRequiresGrant *bool
// shutdownCtx, when non-nil, tells startS3Server to gracefully shut down
// its HTTP/gRPC servers once the ctx is cancelled, in addition to on
// interrupt. Used by weed mini to orchestrate an ordered shutdown.
@@ -140,6 +142,8 @@ func init() {
s3StandaloneOptions.cacheSizeMB = cmdS3.Flag.Int64("cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
s3StandaloneOptions.readerCacheSizeMB = cmdS3.Flag.Int64("readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage)
s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
}
var cmdS3 = &Command{
@@ -337,7 +341,8 @@ func (s3opt *S3Options) startS3Server() bool {
router := mux.NewRouter().SkipClean(true)
var localFilerSocket string
if s3opt.localFilerSocket != nil {
// The filer only listens on its local socket outside Windows.
if s3opt.localFilerSocket != nil && runtime.GOOS != "windows" {
localFilerSocket = *s3opt.localFilerSocket
}
var s3ApiServer *s3api.S3ApiServer
@@ -399,6 +404,8 @@ func (s3opt *S3Options) startS3Server() bool {
MaxMB: filerMaxMB,
AllowUntrustedRemoteEndpoints: *s3opt.allowUntrustedRemoteEndpoints,
AllowBackslashInKey: *s3opt.allowBackslashInKey,
IamSelfServiceRequiresGrant: *s3opt.iamSelfServiceRequiresGrant,
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
weed/command/scaffold/filer.toml
+10
View File
@@ -33,12 +33,22 @@ recursive_delete = false
# faster than previous leveldb, recommended.
enabled = true
dir = "./filerldb2" # directory to store level db files
# optional tuning, applied to each of the 8 partitioned stores
# block_cache_size_mb = 32 # sorted-table read cache
# write_buffer_size_mb = 16 # memtable size before flushing to disk
# block_size_kb = 4 # uncompressed block ('page') size inside table files
# compaction_table_size_mb = 2 # target size of generated table files
[leveldb3]
# similar to leveldb2.
# each bucket has its own meta store.
enabled = false
dir = "./filerldb3" # directory to store level db files
# optional tuning, applied to the default store and each bucket store
# block_cache_size_mb = 32 # sorted-table read cache
# write_buffer_size_mb = 16 # memtable size before flushing to disk
# block_size_kb = 4 # uncompressed block ('page') size inside table files
# compaction_table_size_mb = 2 # target size of generated table files
[rocksdb]
# local on disk, similar to leveldb
weed/command/server.go
+3
View File
@@ -195,10 +195,12 @@ func init() {
s3Options.iamReadOnly = cmdServer.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
s3Options.cipher = cmdServer.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
s3Options.externalUrl = cmdServer.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Advertised to Iceberg and Lance clients, and tried first when verifying S3 signatures behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
s3Options.allowBackslashInKey = cmdServer.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
s3Options.defaultFileMode = cmdServer.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
s3Options.cacheSizeMB = cmdServer.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
@@ -214,6 +216,7 @@ func init() {
sftpOptions.dataCenter = cmdServer.Flag.String("sftp.dataCenter", "", "prefer to read and write to volumes in this data center")
sftpOptions.localSocket = cmdServer.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
iamOptions.selfServiceRequiresGrant = s3Options.iamSelfServiceRequiresGrant
webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")
weed/command/shell.go
+8 -4
View File
@@ -3,6 +3,7 @@ package command
import (
"fmt"
"os"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -12,10 +13,11 @@ import (
)
var (
shellOptions shell.ShellOptions
shellInitialFiler *string
shellCluster *string
shellDebug *bool
shellOptions shell.ShellOptions
shellInitialFiler *string
shellCluster *string
shellDebug *bool
shellConnectTimeout *time.Duration
)
func init() {
@@ -25,6 +27,7 @@ func init() {
shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888")
shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
shellDebug = cmdShell.Flag.Bool("debug", false, "print informational logs to stderr")
shellConnectTimeout = cmdShell.Flag.Duration("connectTimeout", 30*time.Second, "timeout for the initial master connection and filer discovery")
}
var cmdShell = &Command{
@@ -64,6 +67,7 @@ func runShell(command *Command, args []string) bool {
}
shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
shellOptions.Debug = *shellDebug
shellOptions.ConnectTimeout = *shellConnectTimeout
if shellOptions.Debug {
fmt.Fprintf(os.Stderr, "master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
}
weed/command/upload.go
+73 -19
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -31,6 +32,7 @@ type UploadOptions struct {
diskType *string
maxMB *int
usePublicUrl *bool
concurrency *int
}
func init() {
@@ -46,6 +48,7 @@ func init() {
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
upload.concurrency = cmdUpload.Flag.Int("c", 8, "concurrent file upload goroutines when uploading a folder")
}
var cmdUpload = &Command{
@@ -85,6 +88,7 @@ func runUpload(cmd *Command, args []string) bool {
if *upload.dir == "" {
return false
}
var filePaths []string
err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
if err == nil {
if !info.IsDir() {
@@ -93,23 +97,7 @@ func runUpload(cmd *Command, args []string) bool {
return nil
}
}
parts, e := operation.NewFileParts([]string{path})
if e != nil {
return e
}
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
Replication: *upload.replication,
Collection: *upload.collection,
DataCenter: *upload.dataCenter,
Ttl: *upload.ttl,
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
return e
}
filePaths = append(filePaths, path)
}
} else {
fmt.Println(err)
@@ -118,6 +106,72 @@ func runUpload(cmd *Command, args []string) bool {
})
if err != nil {
fmt.Println(err.Error())
}
concurrency := *upload.concurrency
if concurrency < 1 {
concurrency = 1
}
masterFn := func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }
pref := operation.StoragePreference{
Replication: *upload.replication,
Collection: *upload.collection,
DataCenter: *upload.dataCenter,
Ttl: *upload.ttl,
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}
results := make([][]operation.SubmitResult, len(filePaths))
ready := make([]chan struct{}, len(filePaths))
for index := range ready {
ready[index] = make(chan struct{})
}
printDone := make(chan bool, 1)
go func() {
failed := false
for index := range ready {
<-ready[index]
bytes, _ := json.Marshal(results[index])
fmt.Println(string(bytes))
for _, r := range results[index] {
if r.Error != "" {
failed = true
}
}
}
printDone <- failed
}()
sem := make(chan struct{}, concurrency)
var wg sync.WaitGroup
for index, filePath := range filePaths {
sem <- struct{}{}
wg.Add(1)
go func(index int, filePath string) {
defer close(ready[index])
defer wg.Done()
defer func() { <-sem }()
parts, e := operation.NewFileParts([]string{filePath})
if e != nil {
results[index] = []operation.SubmitResult{{FileName: filepath.Base(filePath), Error: e.Error()}}
return
}
fileResults, e := operation.SubmitFiles(masterFn, grpcDialOption, parts, pref, *upload.usePublicUrl)
if e != nil {
for i := range fileResults {
if fileResults[i].Error == "" {
fileResults[i].Error = e.Error()
}
}
}
results[index] = fileResults
}(index, filePath)
}
wg.Wait()
if <-printDone {
return false
}
if err != nil {
return false
}
} else {
@@ -134,12 +188,12 @@ func runUpload(cmd *Command, args []string) bool {
DiskType: *upload.diskType,
MaxMB: *upload.maxMB,
}, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if err != nil {
fmt.Println(err.Error())
return false
}
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
}
return true
}
weed/credential/credential_manager.go
+6 -4
View File
@@ -15,6 +15,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
// FilerAddressSetter is an interface for credential stores that need a dynamic filer address
@@ -168,15 +169,16 @@ func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_
staticNames := cm.staticNames
cm.staticMu.RUnlock()
if len(staticNames) > 0 {
var dynamicOnly []*iam_pb.Identity
for _, ident := range config.Identities {
configCopy := proto.Clone(config).(*iam_pb.S3ApiConfiguration)
dynamicOnly := configCopy.Identities[:0]
for _, ident := range configCopy.Identities {
if !staticNames[ident.Name] {
dynamicOnly = append(dynamicOnly, ident)
}
}
configCopy := *config
clear(configCopy.Identities[len(dynamicOnly):])
configCopy.Identities = dynamicOnly
return cm.Store.SaveConfiguration(ctx, &configCopy)
return cm.Store.SaveConfiguration(ctx, configCopy)
}
return cm.Store.SaveConfiguration(ctx, config)
}
weed/credential/credential_manager_snapshot_test.go
+46
View File
@@ -0,0 +1,46 @@
package credential
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"google.golang.org/protobuf/proto"
)
type snapshotConfigurationStore struct {
CredentialStore
saved *iam_pb.S3ApiConfiguration
}
func (s *snapshotConfigurationStore) SaveConfiguration(_ context.Context, config *iam_pb.S3ApiConfiguration) error {
s.saved = config
return nil
}
func TestSaveConfigurationFiltersStaticIdentitySnapshot(t *testing.T) {
t.Parallel()
store := &snapshotConfigurationStore{}
manager := &CredentialManager{Store: store}
manager.SetStaticIdentities([]*iam_pb.Identity{{Name: "static"}})
source := &iam_pb.S3ApiConfiguration{Identities: []*iam_pb.Identity{
{Name: "alice", Actions: []string{"Read"}},
{Name: "static"},
{Name: "bob", Actions: []string{"Write"}},
}}
before := proto.Clone(source)
if err := manager.SaveConfiguration(context.Background(), source); err != nil {
t.Fatal(err)
}
if store.saved == nil || len(store.saved.Identities) != 2 || store.saved.Identities[0].Name != "alice" || store.saved.Identities[1].Name != "bob" {
t.Fatalf("unexpected saved configuration: %v", store.saved)
}
if !proto.Equal(source, before) {
t.Fatal("filtering mutated the caller's configuration")
}
store.saved.Identities[0].Actions[0] = "Admin"
store.saved.Identities[1].Name = "changed"
if !proto.Equal(source, before) {
t.Fatal("saved identities alias the caller's configuration")
}
}
weed/filer/abstract_sql/abstract_sql_store.go
+11 -2
View File
@@ -23,14 +23,20 @@ type SqlGenerator interface {
GetSqlDeleteFolderChildren(tableName string) string
GetSqlListExclusive(tableName string) string
GetSqlListInclusive(tableName string) string
// GetSqlScanPage returns the query and arguments selecting dirhash, name,
// directory and meta of up to limit rows after the (dirhash, name) cursor.
GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{})
GetSqlCreateTable(tableName string) string
GetSqlDropTable(tableName string) string
}
type AbstractSqlStore struct {
SqlGenerator
DB *sql.DB
KvDB *sql.DB
DB *sql.DB
KvDB *sql.DB
// ScanDB, when set, is the pool snapshot scans run on (SQLite's main pool
// is a single connection that a long scan must not hold).
ScanDB *sql.DB
SupportBucketTable bool
SkipDDL bool
dbs map[string]bool
@@ -457,6 +463,9 @@ func (store *AbstractSqlStore) Shutdown() {
if store.KvDB != nil {
store.KvDB.Close()
}
if store.ScanDB != nil {
store.ScanDB.Close()
}
}
func isValidBucket(bucket string) bool {
weed/filer/abstract_sql/abstract_sql_store_scan.go
+368
View File
@@ -0,0 +1,368 @@
package abstract_sql
import (
"context"
"database/sql"
"fmt"
"math"
"strings"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var _ filer.EntryScanner = (*AbstractSqlStore)(nil)
// scanTable is one table to read during a scan together with the path its
// directory column is relative to: "" for the default table, whose rows hold
// full directories, and "/buckets/<bucket>" for a per-bucket table.
type scanTable struct {
table string
base string
// cachedOnly marks a table that only a stale bucket cache remembers: its
// bucket directory was already gone from the snapshot, so a missing table
// means it was dropped meanwhile, not that rows are absent.
cachedOnly bool
}
// scanPageSize is how many rows one keyset page of a scan fetches.
var scanPageSize = 1000
// SetScanPageSizeForTest lets tests drive the keyset cursor across pages. It
// returns the previous size.
func SetScanPageSizeForTest(n int) int {
prev := scanPageSize
scanPageSize = n
return prev
}
// ScanEntries implements filer.EntryScanner: every table that can hold an
// entry below dirPath is read inside one REPEATABLE READ, read-only
// transaction, in primary-key pages. All rows are seen as of one instant; a
// rename committed meanwhile shows up under exactly one of its names. The
// transaction takes no row locks and does not block writers; it does hold the
// tables it has read against DDL, so a bucket deleted while the scan runs
// drops its table once the scan ends. A page's cursor is closed before its
// entries go to fn, so fn may read the store through the context it receives
// (which carries the transaction) on the same connection.
func (store *AbstractSqlStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
// Read before the transaction holds a connection: OnBucketCreation keeps
// dbsLock while it waits for a connection of its own, so taking the lock
// while holding a connection could deadlock on an exhausted pool.
cachedBuckets := store.cachedBucketNames()
tx, err := store.beginSnapshotTx(ctx)
if err != nil {
return err
}
// Read-only: nothing to commit, and a rollback releases the snapshot.
defer tx.Rollback()
tables, err := store.scanTables(ctx, tx, dirPath, cachedBuckets)
if err != nil {
return err
}
// Reads made through this context (getTxOrDB, kvDB) use the transaction
// and therefore the snapshot.
txCtx := context.WithValue(ctx, "tx", tx)
// The root's own row may live in a table the scan does not read (a
// bucket directory is in the default table, its contents in the bucket's).
if root, err := store.readRowInSnapshot(txCtx, tx, dirPath); err != nil {
return err
} else if root != nil {
if err := fn(txCtx, root); err != nil {
return err
}
}
for _, t := range tables {
if err := store.scanOneTable(txCtx, tx, t, dirPath, fn); err != nil {
return err
}
}
return nil
}
// readRowInSnapshot reads one entry's row inside the transaction, routing to
// the table getTxOrDB would use but without its bucket-table creation. A
// missing row is nil.
func (store *AbstractSqlStore) readRowInSnapshot(ctx context.Context, tx *sql.Tx, p util.FullPath) (*filer.Entry, error) {
if p == "/" {
return nil, nil
}
parent, name := p.DirAndName()
table, shortDir := DEFAULT_TABLE, parent
if store.SupportBucketTable {
if bucket := bucketTableFor(util.FullPath(parent)); bucket != "" {
table = bucket
if shortDir = strings.TrimPrefix(parent, bucketBase(bucket)); shortDir == "" {
shortDir = "/"
}
}
}
var data []byte
err := tx.QueryRowContext(ctx, store.GetSqlFind(table), util.HashStringToLong(shortDir), name, shortDir).Scan(&data)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, missingTableOr(err, table, fmt.Sprintf("read %s", p))
}
entry := &filer.Entry{FullPath: p}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
return nil, fmt.Errorf("decode %s: %v", p, err)
}
return entry, nil
}
// beginSnapshotTx starts the read-only REPEATABLE READ transaction that gives
// one snapshot across statements. Nothing weaker is tried: under READ
// COMMITTED each page would see a different snapshot, and the stream would
// still be advertised as one.
func (store *AbstractSqlStore) beginSnapshotTx(ctx context.Context) (*sql.Tx, error) {
db := store.DB
if store.ScanDB != nil {
db = store.ScanDB
} else if db.Stats().MaxOpenConnections == 1 {
// The scan would hold the filer's only connection for its whole run.
return nil, fmt.Errorf("%w: the store's single connection cannot be held for a scan", filer.ErrUnsupportedEntryScan)
}
// A BeginTx failure (a down database) must fail the traversal, not route it
// to the walk: a recovered database lets the walk read renamed-away entries
// as absent, the very loss this scan exists to prevent.
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if err != nil {
return nil, fmt.Errorf("begin snapshot: %v", err)
}
return tx, nil
}
func (store *AbstractSqlStore) cachedBucketNames() []string {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
names := make([]string, 0, len(store.dbs))
for name := range store.dbs {
names = append(names, name)
}
return names
}
// scanTables picks the tables to read for dirPath: the default table alone,
// one bucket's table for a path inside that bucket, or the default table plus
// every bucket table (enumerated from /buckets inside the snapshot).
func (store *AbstractSqlStore) scanTables(ctx context.Context, tx *sql.Tx, dirPath util.FullPath, cachedBuckets []string) ([]scanTable, error) {
if !store.SupportBucketTable {
return []scanTable{{table: DEFAULT_TABLE}}, nil
}
if bucket := bucketTableFor(dirPath); bucket != "" {
return []scanTable{{table: bucket, base: bucketBase(bucket)}}, nil
}
tables := []scanTable{{table: DEFAULT_TABLE}}
if dirPath != "/" && string(dirPath) != bucketsRoot {
// Outside every bucket (e.g. /topics, or an internal /buckets/.system
// folder): everything below dirPath is in the default table.
return tables, nil
}
buckets, err := store.listBucketNames(ctx, tx, cachedBuckets)
if err != nil {
return nil, err
}
for _, bucket := range buckets.names {
tables = append(tables, scanTable{table: bucket, base: bucketBase(bucket), cachedOnly: !buckets.seen[bucket]})
}
return tables, nil
}
const bucketsRoot = "/buckets"
func bucketBase(bucket string) string {
return bucketsRoot + "/" + bucket
}
// bucketTableFor returns the bucket whose table alone holds everything below
// dirPath, or "" when dirPath is not inside a real bucket (as getTxOrDB routes).
func bucketTableFor(dirPath util.FullPath) string {
p := string(dirPath)
if !strings.HasPrefix(p, bucketsRoot+"/") {
return ""
}
rest := p[len(bucketsRoot)+1:]
bucket := rest
if i := strings.Index(rest, "/"); i >= 0 {
bucket = rest[:i]
}
if bucket == "" || strings.HasPrefix(bucket, ".") || !isValidBucket(bucket) {
return ""
}
return bucket
}
// bucketNames is the snapshot-listed and cached-only buckets whose tables a
// scan may read.
type bucketNames struct {
names []string
seen map[string]bool // listed under /buckets inside the snapshot
}
// listBucketNames reads the directories under /buckets within the snapshot
// and keeps those that route to their own table, plus the buckets this
// process has already seen (their directory entry may be gone, their rows not).
func (store *AbstractSqlStore) listBucketNames(ctx context.Context, tx *sql.Tx, cachedBuckets []string) (bucketNames, error) {
var names bucketNames
names.seen = make(map[string]bool)
added := make(map[string]bool)
add := func(name string) {
if added[name] || strings.HasPrefix(name, ".") || !isValidBucket(name) {
return
}
added[name] = true
names.names = append(names.names, name)
}
const page = 1000
startFrom := ""
for {
type row struct {
name string
data []byte
}
var rows []row
if err := func() error {
res, err := tx.QueryContext(ctx, store.GetSqlListExclusive(DEFAULT_TABLE), util.HashStringToLong(bucketsRoot), startFrom, bucketsRoot, "%", page)
if err != nil {
return fmt.Errorf("list %s: %v", bucketsRoot, err)
}
defer res.Close()
for res.Next() {
var r row
if err := res.Scan(&r.name, &r.data); err != nil {
return fmt.Errorf("scan %s: %v", bucketsRoot, err)
}
rows = append(rows, r)
}
return res.Err()
}(); err != nil {
return bucketNames{}, err
}
for _, r := range rows {
startFrom = r.name
entry := &filer.Entry{FullPath: util.NewFullPath(bucketsRoot, r.name)}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
return bucketNames{}, fmt.Errorf("scan decode %s: %v", entry.FullPath, err)
}
if entry.IsDirectory() {
names.seen[r.name] = true
add(r.name)
}
}
if len(rows) < page {
break
}
}
for _, name := range cachedBuckets {
add(name)
}
return names, nil
}
// scanOneTable reads one table in primary-key pages and hands the entries
// strictly below dirPath to fn (the root itself is read separately). Key-value
// records share the default table; GenDirAndName stores the raw key bytes as
// their dirhash while entry rows hash the directory, so the mismatch
// identifies them exactly.
func (store *AbstractSqlStore) scanOneTable(ctx context.Context, tx *sql.Tx, t scanTable, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
type row struct {
dirHash int64
name, dir string
data []byte
}
cursorHash, cursorName := int64(math.MinInt64), ""
firstPage := true
for {
var page []row
if t.cachedOnly && firstPage {
// A missing table can only be met by the first page read: once a
// row came back the tx holds the table against a drop. On
// PostgreSQL the failed query aborts the transaction, so the skip
// needs a savepoint to roll back to before the next table's read.
if _, err := tx.ExecContext(ctx, "SAVEPOINT seaweed_scan_bucket"); err != nil {
return fmt.Errorf("scan table %s: %v", t.table, err)
}
}
sqlText, args := store.GetSqlScanPage(t.table, cursorHash, cursorName, scanPageSize)
rows, err := tx.QueryContext(ctx, sqlText, args...)
if err != nil {
if t.cachedOnly && isMissingTableError(err) {
tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT seaweed_scan_bucket")
return nil // a cached-only name whose table was dropped meanwhile
}
return missingTableOr(err, t.table, fmt.Sprintf("scan table %s", t.table))
}
firstPage = false
if err := func() error {
defer rows.Close()
for rows.Next() {
var r row
if err := rows.Scan(&r.dirHash, &r.name, &r.dir, &r.data); err != nil {
return fmt.Errorf("scan table %s: %v", t.table, err)
}
page = append(page, r)
}
return rows.Err()
}(); err != nil {
return err
}
for _, r := range page {
cursorHash, cursorName = r.dirHash, r.name
if r.dirHash != util.HashStringToLong(r.dir) {
continue // a key-value record, see GenDirAndName
}
fullPath := util.NewFullPath(t.base+r.dir, r.name)
if !isStrictlyBelow(fullPath, dirPath) {
continue
}
entry := &filer.Entry{FullPath: fullPath}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
return fmt.Errorf("scan decode %s: %v", fullPath, err)
}
if err := fn(ctx, entry); err != nil {
return err
}
}
if len(page) < scanPageSize {
return nil
}
}
}
// missingTableOr names the condition when a bucket's table is not there. The
// scan fails closed: from inside the snapshot a table that was never created
// cannot be told from one dropped a moment ago, whose rows the snapshot still
// counts, and treating either as empty would let fsck purge referenced chunks.
func missingTableOr(err error, table, what string) error {
if table != DEFAULT_TABLE && isMissingTableError(err) {
return fmt.Errorf("bucket %s has a directory entry but no table (never created, or dropped during the scan): %v", table, err)
}
return fmt.Errorf("%s: %v", what, err)
}
// isMissingTableError recognizes a missing table by message, as InsertEntry
// recognizes duplicates: PostgreSQL `relation "x" does not exist` (SQLSTATE
// 42P01), MySQL error 1146 (SQLSTATE 42S02), SQLite "no such table".
func isMissingTableError(err error) bool {
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "42p01") ||
(strings.Contains(msg, "relation \"") && strings.Contains(msg, "does not exist")) ||
strings.Contains(msg, "42s02") ||
strings.Contains(msg, "error 1146") ||
strings.Contains(msg, "no such table")
}
// isStrictlyBelow reports whether p is a descendant of dir (not dir itself).
func isStrictlyBelow(p, dir util.FullPath) bool {
if dir == "/" {
return p != "/"
}
return strings.HasPrefix(string(p), string(dir)+"/")
}
weed/filer/abstract_sql/abstract_sql_store_scan_test.go
+51
View File
@@ -0,0 +1,51 @@
package abstract_sql
import (
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestBucketTableFor(t *testing.T) {
cases := map[string]string{
"/": "",
"/buckets": "",
"/buckets/": "",
"/buckets/mybucket": "mybucket",
"/buckets/mybucket/a/b": "mybucket",
"/buckets/.system/owners": "",
"/buckets/Invalid_Bucket/x": "",
"/topics/.system/log": "",
"/bucketsfoo/mybucket": "",
}
for path, want := range cases {
if got := bucketTableFor(util.FullPath(path)); got != want {
t.Errorf("bucketTableFor(%s) = %q, want %q", path, got, want)
}
}
}
// Only a missing table may be skipped; any other schema error must fail the
// scan, or a bucket's live chunks would be reported as orphans.
func TestIsMissingTableError(t *testing.T) {
for _, msg := range []string{
`ERROR: relation "ghost" does not exist (SQLSTATE 42P01)`,
`Error 1146 (42S02): Table 'seaweedfs.ghost' doesn't exist`,
`SQL logic error: no such table: ghost (1)`,
} {
if !isMissingTableError(errors.New(msg)) {
t.Errorf("not recognized as a missing table: %s", msg)
}
}
for _, msg := range []string{
`ERROR: column "directory" does not exist (SQLSTATE 42703)`,
`Error 1054 (42S22): Unknown column 'directory' in 'field list'`,
`SQL logic error: no such column: directory (1)`,
"connection refused",
} {
if isMissingTableError(errors.New(msg)) {
t.Errorf("wrongly taken for a missing table: %s", msg)
}
}
}
weed/filer/filer.go
+26
View File
@@ -416,6 +416,28 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
return f.Store.RollbackTransaction(ctx)
}
type syncWriteKey struct{}
// WithSyncWrite marks the entry writes under ctx durable-on-ack: a store that
// buffers writes by default must flush them before returning.
func WithSyncWrite(ctx context.Context) context.Context {
return context.WithValue(ctx, syncWriteKey{}, true)
}
// SyncWrite reports whether ctx carries a durability request for entry writes.
func SyncWrite(ctx context.Context) bool {
return ctx.Value(syncWriteKey{}) != nil
}
// syncWriteContext flags ctx when the storage rule on p asks for fsync, so the
// store write is held to the same durability as the data path.
func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context {
if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync {
return WithSyncWrite(ctx)
}
return ctx
}
// CreateEntry creates or replaces an entry. When existing is non-nil the caller
// has already fetched the current entry at this path under a path lock, and it
// is reused instead of looking the store up again; pass nil to have CreateEntry
@@ -426,6 +448,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
return nil
}
ctx = f.syncWriteContext(ctx, entry.FullPath)
if entry.FullPath.IsLongerFileName(maxFilenameLength) {
return filer_pb.ErrEntryNameTooLong
}
@@ -631,6 +655,7 @@ func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath)
// deleted with, so a restore cannot hand back a directory more permissive than the one
// it replaces - including when someone else already put it back with wider ones.
func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error {
ctx = f.syncWriteContext(ctx, dirPath)
existing, err := f.FindEntry(ctx, dirPath)
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return err
@@ -680,6 +705,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
}
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
ctx = f.syncWriteContext(ctx, entry.FullPath)
if oldEntry != nil {
entry.Attr.Crtime = oldEntry.Attr.Crtime
if oldEntry.Attr.Inode != 0 {
weed/filer/filer_delete_entry.go
+2
View File
@@ -122,6 +122,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
return nil
}
ctx = f.syncWriteContext(ctx, p)
entry, findErr := f.FindEntry(ctx, p)
if findErr != nil {
return findErr
weed/filer/filerstore.go
+16
View File
@@ -44,6 +44,22 @@ type FilerStore interface {
Shutdown()
}
// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before
// any entry is visited, when the store cannot serve the scan.
var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store")
// EntryScanner is an optional FilerStore capability: visit every entry at and
// below dirPath, in any order, from one consistent snapshot of the store.
//
// A directory walk sees each directory at a different time, so an entry
// renamed from a directory not yet listed into one already listed is never
// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it
// exactly once. The context handed to fn carries the snapshot, so store reads
// made with it (hard-link records) see the same instant.
type EntryScanner interface {
ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error
}
type BucketAware interface {
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
weed/filer/filerstore_wrapper.go
+29
View File
@@ -2,6 +2,7 @@ package filer
import (
"context"
"errors"
"fmt"
"io"
"math"
@@ -19,6 +20,7 @@ import (
var (
_ = VirtualFilerStore(&FilerStoreWrapper{})
_ = Debuggable(&FilerStoreWrapper{})
_ = EntryScanner(&FilerStoreWrapper{})
)
type VirtualFilerStore interface {
@@ -402,6 +404,33 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
return lastFileName, err
}
// ScanEntries delegates to the default store. With path-specific stores
// configured, entries below dirPath could live in several stores and one
// snapshot per store is not one snapshot, so the scan is declined. Hard links
// are resolved inside the snapshot; a store error fails the scan (the per-path
// copy can be stale), a missing record falls back to it as listings do.
func (fsw *FilerStoreWrapper) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error {
if fsw.hasPathSpecificStore {
return ErrUnsupportedEntryScan
}
scanner, ok := fsw.defaultStore.(EntryScanner)
if !ok {
return ErrUnsupportedEntryScan
}
stats.FilerStoreCounter.WithLabelValues(fsw.defaultStore.GetName(), "scan").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.defaultStore.GetName(), "scan").Observe(time.Since(start).Seconds())
}()
return scanner.ScanEntries(ctx, dirPath, func(scanCtx context.Context, entry *Entry) error {
if err := fsw.maybeReadHardLink(scanCtx, entry); err != nil && !errors.Is(err, ErrKvNotFound) {
return fmt.Errorf("resolve hard link of %s: %w", entry.FullPath, err)
}
filer_pb.AfterEntryDeserialization(entry.GetChunks())
return fn(scanCtx, entry)
})
}
func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
actualStore := fsw.getActualStore(dirPath + "/")
weed/filer/leveldb/leveldb_store.go
+12 -3
View File
@@ -79,6 +79,15 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
key := genKey(entry.DirAndName())
@@ -91,7 +100,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = store.db.Put(key, value, nil)
err = store.db.Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -168,7 +177,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful
func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
key := genKey(fullpath.DirAndName())
err = store.db.Delete(key, nil)
err = store.db.Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -195,7 +204,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
}
iter.Release()
err = store.db.Write(batch, nil)
err = store.db.Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
weed/filer/leveldb2/leveldb2_store.go
+64 -9
View File
@@ -6,6 +6,7 @@ import (
"crypto/md5"
"fmt"
"io"
"math"
"os"
"github.com/syndtr/goleveldb/leveldb"
@@ -28,6 +29,11 @@ type LevelDB2Store struct {
dbs []*leveldb.DB
dbCount int
ReadOnly bool
blockCacheSizeMB int
writeBufferSizeMB int
blockSizeKB int
compactionTableSizeMB int
}
func (store *LevelDB2Store) GetName() string {
@@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string {
func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
dir := configuration.GetString(prefix + "dir")
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
if err := store.validateSizeOptions(); err != nil {
return err
}
return store.initialize(dir, 8)
}
// validateSizeOptions rejects sizes too large to convert to bytes without
// overflowing int on this platform.
func (store *LevelDB2Store) validateSizeOptions() error {
for name, kv := range map[string][2]int{
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
"block_size_kb": {store.blockSizeKB, 1 << 10},
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
} {
if kv[0] > math.MaxInt/kv[1] {
return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0])
}
}
return nil
}
func (store *LevelDB2Store) buildOptions() *opt.Options {
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
ReadOnly: store.ReadOnly,
}
if store.blockCacheSizeMB > 0 {
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
}
if store.writeBufferSizeMB > 0 {
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
}
if store.blockSizeKB > 0 {
opts.BlockSize = store.blockSizeKB * 1024
}
if store.compactionTableSizeMB > 0 {
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
}
return opts
}
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
glog.Infof("filer store leveldb2 dir: %s", dir)
os.MkdirAll(dir, 0755)
@@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
}
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
ReadOnly: store.ReadOnly,
}
opts := store.buildOptions()
for d := 0; d < dbCount; d++ {
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
@@ -81,6 +127,15 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
dir, name := entry.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
@@ -94,7 +149,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = store.dbs[partitionId].Put(key, value, nil)
err = store.dbs[partitionId].Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -140,7 +195,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
dir, name := fullpath.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
err = store.dbs[partitionId].Delete(key, nil)
err = store.dbs[partitionId].Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -167,7 +222,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
}
iter.Release()
err = store.dbs[partitionId].Write(batch, nil)
err = store.dbs[partitionId].Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
weed/filer/leveldb2/leveldb2_store_test.go
+89
View File
@@ -2,6 +2,8 @@ package leveldb
import (
"context"
"math"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) {
}
type fakeConfiguration map[string]interface{}
func (c fakeConfiguration) GetString(key string) string {
value, _ := c[key].(string)
return value
}
func (c fakeConfiguration) GetBool(key string) bool {
value, _ := c[key].(bool)
return value
}
func (c fakeConfiguration) GetInt(key string) int {
value, _ := c[key].(int)
return value
}
func (c fakeConfiguration) GetStringSlice(key string) []string {
value, _ := c[key].([]string)
return value
}
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
if _, found := c[key]; !found {
c[key] = value
}
}
func TestInitializeOptions(t *testing.T) {
store := &LevelDB2Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.block_cache_size_mb": 64,
"leveldb2.write_buffer_size_mb": 32,
"leveldb2.block_size_kb": 16,
"leveldb2.compaction_table_size_mb": 8,
}, "leveldb2."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions()
if opts.BlockCacheCapacity != 64*1024*1024 {
t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 32*1024*1024 {
t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer)
}
if opts.BlockSize != 16*1024 {
t.Errorf("block size %d, want 16KiB", opts.BlockSize)
}
if opts.CompactionTableSize != 8*1024*1024 {
t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize)
}
}
func TestInitializeDefaultOptions(t *testing.T) {
store := &LevelDB2Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.block_cache_size_mb": -1,
"leveldb2.write_buffer_size_mb": 0,
}, "leveldb2."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions()
if opts.BlockCacheCapacity != 32*1024*1024 {
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 16*1024*1024 {
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
}
}
func TestInitializeRejectsOversizedOption(t *testing.T) {
store := &LevelDB2Store{}
err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
}, "leveldb2.")
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
t.Fatalf("expected overflow rejection, got %v", err)
}
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
weed/filer/leveldb3/leveldb3_store.go
+68 -18
View File
@@ -6,6 +6,7 @@ import (
"crypto/md5"
"fmt"
"io"
"math"
"os"
"strings"
"sync"
@@ -35,6 +36,11 @@ type LevelDB3Store struct {
dbs map[string]*leveldb.DB
dbsLock sync.RWMutex
ReadOnly bool
blockCacheSizeMB int
writeBufferSizeMB int
blockSizeKB int
compactionTableSizeMB int
}
func (store *LevelDB3Store) GetName() string {
@@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string {
func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
dir := configuration.GetString(prefix + "dir")
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
if err := store.validateSizeOptions(); err != nil {
return err
}
return store.initialize(dir)
}
// validateSizeOptions rejects sizes too large to convert to bytes without
// overflowing int on this platform.
func (store *LevelDB3Store) validateSizeOptions() error {
for name, kv := range map[string][2]int{
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
"block_size_kb": {store.blockSizeKB, 1 << 10},
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
} {
if kv[0] > math.MaxInt/kv[1] {
return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0])
}
}
return nil
}
func (store *LevelDB3Store) buildOptions(name string) *opt.Options {
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8),
ReadOnly: store.ReadOnly,
}
if name != DEFAULT {
opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB
opts.WriteBuffer = 8 * 1024 * 1024 // default value is 4MiB
}
if store.blockCacheSizeMB > 0 {
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
}
if store.writeBufferSizeMB > 0 {
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
}
if store.blockSizeKB > 0 {
opts.BlockSize = store.blockSizeKB * 1024
}
if store.compactionTableSizeMB > 0 {
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
}
return opts
}
func (store *LevelDB3Store) initialize(dir string) (err error) {
glog.Infof("filer store leveldb3 dir: %s", dir)
os.MkdirAll(dir, 0755)
@@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
}
func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
bloom := filter.NewBloomFilter(8) // false positive rate 0.02
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: bloom,
ReadOnly: store.ReadOnly,
}
if name != DEFAULT {
opts = &opt.Options{
BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB
Filter: bloom,
ReadOnly: store.ReadOnly,
}
}
opts := store.buildOptions(name)
dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
os.MkdirAll(dbFolder, 0755)
@@ -180,6 +221,15 @@ func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
db, _, shortPath, err := store.findDB(entry.FullPath, false)
@@ -199,7 +249,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = db.Put(key, value, nil)
err = db.Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -257,7 +307,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
dir, name := shortPath.DirAndName()
key := genKey(dir, name)
err = db.Delete(key, nil)
err = db.Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -298,7 +348,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w
}
iter.Release()
err = db.Write(batch, nil)
err = db.Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
weed/filer/leveldb3/leveldb3_store_test.go
+99
View File
@@ -2,6 +2,8 @@ package leveldb
import (
"context"
"math"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) {
}
type fakeConfiguration map[string]interface{}
func (c fakeConfiguration) GetString(key string) string {
value, _ := c[key].(string)
return value
}
func (c fakeConfiguration) GetBool(key string) bool {
value, _ := c[key].(bool)
return value
}
func (c fakeConfiguration) GetInt(key string) int {
value, _ := c[key].(int)
return value
}
func (c fakeConfiguration) GetStringSlice(key string) []string {
value, _ := c[key].([]string)
return value
}
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
if _, found := c[key]; !found {
c[key] = value
}
}
func TestInitializeOptions(t *testing.T) {
store := &LevelDB3Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.block_cache_size_mb": 64,
"leveldb3.write_buffer_size_mb": 32,
"leveldb3.block_size_kb": 16,
"leveldb3.compaction_table_size_mb": 8,
}, "leveldb3."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
for _, name := range []string{DEFAULT, "bucket-a"} {
opts := store.buildOptions(name)
if opts.BlockCacheCapacity != 64*1024*1024 {
t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 32*1024*1024 {
t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer)
}
if opts.BlockSize != 16*1024 {
t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize)
}
if opts.CompactionTableSize != 8*1024*1024 {
t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize)
}
}
}
func TestInitializeDefaultOptions(t *testing.T) {
store := &LevelDB3Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.block_cache_size_mb": -1,
"leveldb3.write_buffer_size_mb": 0,
}, "leveldb3."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions(DEFAULT)
if opts.BlockCacheCapacity != 32*1024*1024 {
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 16*1024*1024 {
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
}
bucketOpts := store.buildOptions("bucket-a")
if bucketOpts.BlockCacheCapacity != 16*1024*1024 {
t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity)
}
if bucketOpts.WriteBuffer != 8*1024*1024 {
t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer)
}
}
func TestInitializeRejectsOversizedOption(t *testing.T) {
store := &LevelDB3Store{}
err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
}, "leveldb3.")
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
t.Fatalf("expected overflow rejection, got %v", err)
}
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
weed/filer/mysql/mysql_sql_gen.go
+9
View File
@@ -62,6 +62,15 @@ func (gen *SqlGenMysql) nameExpr() string {
return "`name`"
}
// GetSqlScanPage spells the cursor as two primary-key ranges: MySQL 5.7 does
// not turn a row-value comparison into a range seek. The name keeps the
// column's own collation (BINARY would filesort every page); the scan promises
// no lexical order. SQLite overrides this method.
func (gen *SqlGenMysql) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE `dirhash` > ? OR (`dirhash` = ? AND `name` > ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
[]interface{}{dirhash, dirhash, name, limit}
}
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
weed/filer/postgres/postgres_sql_gen.go
+8
View File
@@ -75,6 +75,14 @@ func (gen *SqlGenPostgres) nameExpr() string {
return "name"
}
// GetSqlScanPage uses a row-value comparison, one primary-key range seek on
// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would
// sort every page); the scan promises no lexical order.
func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName),
[]interface{}{dirhash, name, limit}
}
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
weed/filer/sqlite/sqlite_store.go
+32 -2
View File
@@ -30,6 +30,31 @@ func (store *SqliteStore) GetName() string {
return "sqlite"
}
// sqliteSqlGen is the MySQL generator with SQLite's own page cursor: SQLite
// seeks a row-value comparison on the primary key but scans the OR form.
type sqliteSqlGen struct {
*mysql.SqlGenMysql
}
func (gen sqliteSqlGen) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE (`dirhash`, `name`) > (?, ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
[]interface{}{dirhash, name, limit}
}
// ScanEntries offers the snapshot scan only in WAL mode: with a rollback
// journal a long read transaction holds a shared lock and writers fail after
// the busy timeout.
func (store *SqliteStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
var mode string
if err := store.DB.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil {
return fmt.Errorf("%w: journal_mode: %v", filer.ErrUnsupportedEntryScan, err)
}
if !strings.EqualFold(mode, "wal") {
return fmt.Errorf("%w: sqlite journal_mode is %q, the scan needs wal", filer.ErrUnsupportedEntryScan, mode)
}
return store.AbstractSqlStore.ScanEntries(ctx, dirPath, fn)
}
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) {
dbFile := configuration.GetString(prefix + "dbFile")
createTable := `CREATE TABLE IF NOT EXISTS "%s" (
@@ -69,11 +94,11 @@ func sqliteDSN(dbFile string) string {
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{
store.SqlGenerator = sqliteSqlGen{&mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table if exists `%s`",
UpsertQueryTemplate: upsertQuery,
}
}}
dsn := sqliteDSN(dbFile)
@@ -94,6 +119,11 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
}, 1, 1, 0); err != nil {
return err
}
// The main pool is one connection, which a long scan must not hold.
if store.ScanDB, err = sql.Open("sqlite", dsn); err != nil {
return fmt.Errorf("open scan connection to %s: %v", dbFile, err)
}
store.ScanDB.SetMaxOpenConns(1)
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
weed/filer/sqlite/sqlite_store_scan_test.go
+288
View File
@@ -0,0 +1,288 @@
//go:build (linux || darwin || windows) && sqlite
package sqlite
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// A rename committed while a scan is running is seen under exactly one of its
// names. The scan reads from one snapshot, so unlike a directory walk it can
// neither miss the entry when it moves into a directory already listed, nor
// count it twice when it moves into one not listed yet.
func TestScanEntriesSeesConcurrentRenameExactlyOnce(t *testing.T) {
dbFile := filepath.Join(t.TempDir(), "filer.db")
reader := &SqliteStore{}
if err := reader.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer reader.Shutdown()
// WAL lets the writer commit while the scan's read transaction is open.
if _, err := reader.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
writer := &SqliteStore{}
if err := writer.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer writer.Shutdown()
ctx := context.Background()
mk := func(p string) *filer.Entry {
return &filer.Entry{FullPath: util.FullPath(p), Attr: filer.Attr{Mode: 0644, Mtime: time.Now(), Crtime: time.Now()}}
}
for _, p := range []string{"/ckpt/a", "/ckpt/b", "/ckpt/tmp/x"} {
if err := reader.InsertEntry(ctx, mk(p)); err != nil {
t.Fatal(err)
}
}
defer func(n int) { abstract_sql.SetScanPageSizeForTest(n) }(abstract_sql.SetScanPageSizeForTest(2))
seen := map[string]int{}
renamed := false
err := reader.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
seen[string(e.FullPath)]++
if renamed {
return nil
}
renamed = true
// /ckpt/tmp/x -> /ckpt/x, committed by another connection mid-scan.
if err := writer.InsertEntry(ctx, mk("/ckpt/x")); err != nil {
return err
}
return writer.DeleteEntry(ctx, util.FullPath("/ckpt/tmp/x"))
})
if err != nil {
t.Fatal(err)
}
if !renamed {
t.Fatal("scan yielded nothing")
}
if seen["/ckpt/a"] != 1 || seen["/ckpt/b"] != 1 {
t.Errorf("untouched entries seen %d and %d times, want once each", seen["/ckpt/a"], seen["/ckpt/b"])
}
if old, new := seen["/ckpt/tmp/x"], seen["/ckpt/x"]; old+new != 1 {
t.Errorf("renamed entry seen %d time(s) under the old name and %d under the new, want exactly one in total", old, new)
} else if old != 1 {
t.Errorf("the rename committed after the snapshot was taken, so the scan must show the old name")
}
if _, err := reader.FindEntry(ctx, util.FullPath("/ckpt/x")); err != nil {
t.Errorf("once the scan is over the new name must be visible: %v", err)
}
}
// Rows that are not entries, key-value records and a regular file under
// /buckets with a bucket-like name, are stepped over; a scan rooted at a bucket
// yields the bucket's own row (default table) and its contents (bucket table).
func TestScanEntriesRowsAndBucketTables(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
ctx := context.Background()
now := time.Now()
// The filer creates bucket tables on bucket creation; the store layer alone does not.
if err := store.CreateTable(ctx, "realbucket"); err != nil {
t.Fatal(err)
}
for _, e := range []*filer.Entry{
{FullPath: "/buckets", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
{FullPath: "/buckets/realbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
{FullPath: "/buckets/realbucket/f1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
{FullPath: "/buckets/readme", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
{FullPath: "/topics/t1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
} {
if err := store.InsertEntry(ctx, e); err != nil {
t.Fatal(err)
}
}
for _, key := range []string{"filer.store.id", "/looks/like/a/path", "x"} {
if err := store.KvPut(ctx, []byte(key), []byte("opaque value, not an Entry")); err != nil {
t.Fatal(err)
}
}
collect := func(root string) []string {
var got []string
if err := store.ScanEntries(ctx, util.FullPath(root), func(_ context.Context, e *filer.Entry) error {
got = append(got, string(e.FullPath))
return nil
}); err != nil {
t.Fatalf("scan %s: %v", root, err)
}
sort.Strings(got)
return got
}
if got, want := collect("/"), []string{"/buckets", "/buckets/readme", "/buckets/realbucket", "/buckets/realbucket/f1", "/topics/t1"}; !reflect.DeepEqual(got, want) {
t.Errorf("scan /: got %v, want %v", got, want)
}
if got, want := collect("/buckets/realbucket"), []string{"/buckets/realbucket", "/buckets/realbucket/f1"}; !reflect.DeepEqual(got, want) {
t.Errorf("scan /buckets/realbucket: got %v, want %v", got, want)
}
// A bucket directory whose table does not exist fails the scan: from the
// snapshot it cannot be told from a table dropped a moment ago.
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/buckets/ghostbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}); err != nil {
t.Fatal(err)
}
err := store.ScanEntries(ctx, "/", func(context.Context, *filer.Entry) error { return nil })
if err == nil || !strings.Contains(err.Error(), "ghostbucket") {
t.Fatalf("scan over a bucket without a table must fail naming it, got %v", err)
}
}
// Without WAL the store declines the scan rather than block writers for its
// whole duration.
func TestScanEntriesDeclinedWithoutWAL(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
}
}
// A store whose pool is one connection, and no scan connection of its own,
// declines rather than hold that connection for the whole scan.
func TestScanEntriesDeclinedOnSingleConnectionPool(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
store.ScanDB.Close()
store.ScanDB = nil
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
}
}
// A hard-linked entry's chunks live in the shared key-value record, not in
// its path row; the wrapper resolves them from inside the snapshot, on the
// single connection the scan holds.
func TestScanEntriesResolvesHardLinks(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
wrapper := filer.NewFilerStoreWrapper(store)
ctx := context.Background()
now := time.Now()
linked := &filer.Entry{
FullPath: "/buckets/hlbucket/f0001",
Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now},
HardLinkId: filer.HardLinkId("hardlink-00000001"),
HardLinkCounter: 2,
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 4096}},
}
if err := wrapper.InsertEntry(ctx, linked); err != nil {
t.Fatal(err)
}
var got *filer.Entry
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
if e.FullPath == linked.FullPath {
got = e
}
return nil
}); err != nil {
t.Fatal(err)
}
if got == nil || len(got.GetChunks()) != 1 || got.GetChunks()[0].FileId != "3,01637037d6" || got.HardLinkCounter != 2 {
t.Fatalf("hard-linked entry not resolved from its key-value record: %+v", got)
}
// Once the record is gone the path row is all there is; the scan hands it
// over instead of failing on that entry every time it runs.
if err := store.KvDelete(ctx, []byte(linked.HardLinkId)); err != nil {
t.Fatal(err)
}
got = nil
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
if e.FullPath == linked.FullPath {
got = e
}
return nil
}); err != nil {
t.Fatalf("a missing hard-link record must not fail the scan: %v", err)
}
if got == nil {
t.Fatal("entry with a missing hard-link record not handed over")
}
}
// The keyset page must be a primary-key seek; an OR-shaped cursor predicate
// makes SQLite scan the table from the start for every page.
func TestScanPageQuerySeeksPrimaryKey(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
sqlText, args := store.GetSqlScanPage("filemeta", int64(-1), "", 10)
rows, err := store.DB.Query("EXPLAIN QUERY PLAN "+sqlText, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "SEARCH") || strings.Contains(joined, "SCAN") {
t.Fatalf("page query does not seek the primary key: %s", joined)
}
}
// A bucket name only the process cache remembers may have its table dropped
// meanwhile; the scan skips it instead of failing.
func TestScanEntriesSkipsCachedOnlyBucketTable(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
store.OnBucketCreation("gonebucket") // the table is never created
if err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }); err != nil {
t.Fatalf("cached-only bucket with a dropped table must not fail the scan: %v", err)
}
}
weed/iam/helpers.go
+8 -5
View File
@@ -98,11 +98,13 @@ var fineGrainedActionMap = map[string]string{
// prefixed (s3:Action) and unprefixed (Action) variants in init().
var baseS3ActionMap = map[string]string{
// Object operations
"DeleteObject": s3_constants.ACTION_WRITE,
"PutObject": s3_constants.ACTION_WRITE,
"GetObject": s3_constants.ACTION_READ,
"DeleteObjectVersion": s3_constants.ACTION_WRITE,
"GetObjectVersion": s3_constants.ACTION_READ,
"DeleteObject": s3_constants.ACTION_WRITE,
"PutObject": s3_constants.ACTION_WRITE,
"GetObject": s3_constants.ACTION_READ,
"DeleteObjectVersion": s3_constants.ACTION_WRITE,
"GetObjectVersion": s3_constants.ACTION_READ,
"GetObjectAttributes": s3_constants.ACTION_READ,
"GetObjectVersionAttributes": s3_constants.ACTION_READ,
// Tagging operations
"GetObjectTagging": s3_constants.ACTION_TAGGING,
"GetObjectVersionTagging": s3_constants.ACTION_TAGGING,
@@ -115,6 +117,7 @@ var baseS3ActionMap = map[string]string{
"DeleteBucketTagging": s3_constants.ACTION_TAGGING,
// ACL operations
"PutObjectAcl": s3_constants.ACTION_WRITE_ACP,
"PutObjectVersionAcl": s3_constants.ACTION_WRITE_ACP,
"GetObjectAcl": s3_constants.ACTION_READ_ACP,
"GetObjectVersionAcl": s3_constants.ACTION_READ_ACP,
"PutBucketAcl": s3_constants.ACTION_WRITE_ACP,
weed/iam/integration/iam_manager.go
+123 -5
View File
@@ -53,7 +53,13 @@ type IAMManager struct {
oidcRetryAgain bool
// oidcRefreshMu serializes refreshes from reading the store to handing
// STS the result, so an older snapshot cannot replace a newer one.
oidcRefreshMu sync.Mutex
oidcRefreshMu sync.Mutex
// oidcManagedProviders caches providers refreshOIDCProvidersFrom built,
// keyed by record identity, so an unchanged record reuses its provider:
// rebuilding would drop the provider's consumed-token (jti) replay cache
// and let an already-used token be presented again. Guarded by
// oidcRefreshMu.
oidcManagedProviders map[string]*oidcManagedProvider
oidcAuditSink OIDCProviderAuditSink
revocationStore SessionRevocationStore
filerAddressProvider func() string // Function to get current filer address
@@ -943,14 +949,65 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
return fmt.Errorf("list OIDC providers: the store was replaced during the refresh")
}
byIssuer := make(map[string][]sts.ScopedOIDCProvider, len(records))
if m.oidcManagedProviders == nil {
m.oidcManagedProviders = make(map[string]*oidcManagedProvider)
}
seen := make(map[string]struct{}, len(records))
// Records for one issuer with identical validation settings share a single
// provider — and with it the consumed-token (jti) replay store — so a token
// spent through one account's record stays spent through the other's too.
sharedProviders := make(map[string]*oidc.OIDCProvider)
for _, rec := range records {
if rec == nil || rec.URL == "" {
continue
}
provider, err := buildOIDCProviderFromRecord(rec)
if err != nil {
glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err)
continue
key := rec.ARN
if key == "" {
key = rec.URL + "\x00" + rec.AccountID
}
seen[key] = struct{}{}
fingerprint := oidcProviderFingerprint(rec)
configFingerprint := oidcProviderConfigFingerprint(rec)
var provider *oidc.OIDCProvider
cached := m.oidcManagedProviders[key]
if shared, ok := sharedProviders[configFingerprint]; ok {
provider = shared
if cached != nil && cached.provider != nil && cached.provider != shared {
// The record's own provider is superseded; its consumed
// tokens move to the shared store so none reopen for replay.
shared.MergeReplayStateFrom(cached.provider)
}
m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
if cached != nil && cached.provider != nil && cached.provider != shared && !m.oidcProviderStillReferenced(cached.provider) {
if err := cached.provider.Shutdown(ctx); err != nil {
glog.Warningf("shutdown superseded OIDC provider %s: %v", rec.ARN, err)
}
}
} else if cached != nil && cached.fingerprint == fingerprint {
provider = cached.provider
sharedProviders[configFingerprint] = provider
} else {
var err error
provider, err = buildOIDCProviderFromRecord(rec)
if err != nil {
delete(m.oidcManagedProviders, key)
glog.Warningf("skip refreshing OIDC provider %s: %v", rec.ARN, err)
continue
}
if cached != nil && cached.provider != nil {
// The rebuild must keep rejecting tokens the replaced
// provider consumed; carry its replay store over, then stop
// its cleanup worker so only the new one sweeps the store —
// unless another record still references that provider.
provider.AdoptReplayStateFrom(cached.provider)
}
sharedProviders[configFingerprint] = provider
m.oidcManagedProviders[key] = &oidcManagedProvider{fingerprint: fingerprint, provider: provider}
if cached != nil && cached.provider != nil && !m.oidcProviderStillReferenced(cached.provider) {
if err := cached.provider.Shutdown(ctx); err != nil {
glog.Warningf("shutdown replaced OIDC provider %s: %v", rec.ARN, err)
}
}
}
// Multiple records may share an issuer when each is scoped to a
// different account; STS picks the right one at validation time
@@ -960,10 +1017,71 @@ func (m *IAMManager) refreshOIDCProvidersFrom(ctx context.Context, store OIDCPro
Provider: provider,
})
}
for key, managed := range m.oidcManagedProviders {
if _, ok := seen[key]; !ok {
delete(m.oidcManagedProviders, key)
// A removed record's provider may be shared by records that are
// still present; shut it down only when nothing references it.
if managed.provider != nil && !m.oidcProviderStillReferenced(managed.provider) {
if err := managed.provider.Shutdown(ctx); err != nil {
glog.Warningf("shutdown removed OIDC provider %s: %v", key, err)
}
}
}
}
m.stsService.SetIAMManagedOIDCProviders(byIssuer)
return nil
}
// oidcManagedProvider is a runtime provider together with the fingerprint of
// the record it was built from.
type oidcManagedProvider struct {
fingerprint string
provider *oidc.OIDCProvider
}
// oidcProviderStillReferenced reports whether any managed record still points
// at the provider, so a shared provider is not shut down with a removed record.
func (m *IAMManager) oidcProviderStillReferenced(provider *oidc.OIDCProvider) bool {
for _, managed := range m.oidcManagedProviders {
if managed.provider == provider {
return true
}
}
return false
}
// oidcProviderFingerprint identifies the record content a provider was built
// from, so a refresh rebuilds the provider only when that content changed.
func oidcProviderFingerprint(rec *OIDCProviderRecord) string {
key, _ := json.Marshal(struct {
ARN string
AccountID string
URL string
ClientIDs []string
Thumbprints []string
AllowedPrincipalTagKeys []string
PolicyClaim string
}{rec.ARN, rec.AccountID, rec.URL, rec.ClientIDs, rec.Thumbprints, rec.AllowedPrincipalTagKeys, rec.PolicyClaim})
sum := sha256.Sum256(key)
return hex.EncodeToString(sum[:])
}
// oidcProviderConfigFingerprint identifies the validation settings a provider
// is built from — everything buildOIDCProviderFromRecord consumes except the
// record's name — so records that differ only in identity reuse one provider.
func oidcProviderConfigFingerprint(rec *OIDCProviderRecord) string {
key, _ := json.Marshal(&oidc.OIDCConfig{
Issuer: rec.URL,
ClientIDs: rec.ClientIDs,
Thumbprints: rec.Thumbprints,
AllowedPrincipalTagKeys: rec.AllowedPrincipalTagKeys,
PolicyClaim: rec.PolicyClaim,
})
sum := sha256.Sum256(key)
return hex.EncodeToString(sum[:])
}
// buildOIDCProviderFromRecord turns a stored record into a runtime
// OIDCProvider. The provider name is the ARN so re-registration is
// idempotent and never collides with static-config entries.
weed/iam/integration/oidc_provider_persist_test.go
+90
View File
@@ -559,3 +559,93 @@ func TestAFailureDuringARetryIsNotDropped(t *testing.T) {
}
assert.GreaterOrEqual(t, store.successes(), 2, "the retry ended on its first success, dropping the refresh that failed while it ran")
}
// A refresh must not rebuild a provider whose record is unchanged: rebuilding
// would drop the provider's consumed-token (jti) replay cache and let an
// already-used token be presented again.
func TestRefreshReusesProviderWhileRecordIsUnchanged(t *testing.T) {
ctx := context.Background()
store := NewMemoryOIDCProviderStore()
mgr := startServer(t, store)
arn := arnOf(t, persistTestAPIIssuer)
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud"},
}))
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
first := mgr.oidcManagedProviders[arn].provider
require.NotNil(t, first)
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
assert.Same(t, first, mgr.oidcManagedProviders[arn].provider,
"an unchanged record was rebuilt, dropping its replay cache")
// A changed record must rebuild so the new configuration applies.
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
ARN: arn, URL: persistTestAPIIssuer, ClientIDs: []string{"aud", "aud2"},
}))
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
rebuilt := mgr.oidcManagedProviders[arn].provider
require.NotNil(t, rebuilt)
assert.NotSame(t, first, rebuilt, "a changed record kept the old provider")
// Deleting the record drops the provider from the cache and the STS map.
require.NoError(t, store.DeleteProvider(ctx, "", arn))
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
assert.NotContains(t, mgr.oidcManagedProviders, arn)
assert.False(t, stsKnowsIssuer(t, mgr, persistTestAPIIssuer))
}
// stsKnowsIssuerForAccount is stsKnowsIssuer through the account-scoped
// lookup used for providers that have no global record.
func stsKnowsIssuerForAccount(t *testing.T, mgr *IAMManager, issuer, accountID string) bool {
t.Helper()
tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"iss": issuer, "sub": "probe", "aud": "aud", "exp": time.Now().Add(time.Hour).Unix(),
}).SignedString([]byte("not-the-providers-key"))
require.NoError(t, err)
_, _, err = mgr.GetSTSService().ValidateWebIdentityTokenForAccount(context.Background(), tok, accountID)
require.Error(t, err, "an unsigned-by-provider token for %s was accepted", issuer)
return !strings.Contains(err.Error(), "no identity provider registered")
}
// Two records for one issuer with identical validation settings share a
// single provider — and with it the consumed-token (jti) replay store — so a
// token spent through one account's record stays spent through the other's.
// Removing one record of the pair must not shut the shared provider down.
func TestSameIssuerRecordsShareOneProvider(t *testing.T) {
ctx := context.Background()
store := NewMemoryOIDCProviderStore()
mgr := startServer(t, store)
arnA := arnOf(t, persistTestAPIIssuer) + "-account-a"
arnB := arnOf(t, persistTestAPIIssuer) + "-account-b"
for _, rec := range []*OIDCProviderRecord{
{ARN: arnA, URL: persistTestAPIIssuer, AccountID: "111111111111", ClientIDs: []string{"aud"}},
{ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"aud"}},
} {
require.NoError(t, store.StoreProvider(ctx, "", rec))
}
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
shared := mgr.oidcManagedProviders[arnA].provider
require.NotNil(t, shared)
assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
"identical records for one issuer built separate replay stores")
// Removing one record of the pair keeps the shared provider alive.
require.NoError(t, store.DeleteProvider(ctx, "", arnA))
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
assert.Same(t, shared, mgr.oidcManagedProviders[arnB].provider,
"a removed sibling record shut down the shared provider")
assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
// The remaining record diverging rebuilds its own provider.
require.NoError(t, store.StoreProvider(ctx, "", &OIDCProviderRecord{
ARN: arnB, URL: persistTestAPIIssuer, AccountID: "222222222222", ClientIDs: []string{"b-aud"},
}))
require.NoError(t, mgr.refreshOIDCProvidersFrom(ctx, store))
assert.NotSame(t, shared, mgr.oidcManagedProviders[arnB].provider,
"a changed record kept the shared provider")
assert.True(t, stsKnowsIssuerForAccount(t, mgr, persistTestAPIIssuer, "222222222222"))
}
weed/iam/oidc/mock_provider.go
+11
View File
@@ -75,6 +75,17 @@ func (m *MockOIDCProvider) Authenticate(ctx context.Context, token string) (*pro
return identity, nil
}
// AuthenticateSingleUse stays on the mock path: the embedded provider's
// method would bypass the mock's test data and demand a real JWKS.
func (m *MockOIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
return m.Authenticate(ctx, token)
}
// ValidateTokenSingleUse stays on the mock path; see AuthenticateSingleUse.
func (m *MockOIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
return m.ValidateToken(ctx, token)
}
// ValidateToken validates tokens using test data
func (m *MockOIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
if !m.initialized {
weed/iam/oidc/oidc_provider.go
+423 -2
View File
@@ -6,6 +6,7 @@ import (
"crypto/elliptic"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/base64"
@@ -19,6 +20,7 @@ import (
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/golang-jwt/jwt/v5"
@@ -27,7 +29,44 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
)
// OIDCProvider implements OpenID Connect authentication
const (
// clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519)
clockSkewTolerance = 5 * time.Minute
// defaultJTICacheSize is the default maximum number of JTI entries to cache
defaultJTICacheSize = 10000
// defaultCleanupInterval is how often to scan for expired JTI entries
defaultCleanupInterval = 5 * time.Minute
// eagerCleanupMinInterval rate limits opportunistic sweeps near capacity
eagerCleanupMinInterval = 5 * time.Second
// tokenFingerprintPrefix marks replay-cache keys derived from the token's
// own fingerprint, used when the token carries no usable jti claim
tokenFingerprintPrefix = "fp:"
)
// jtiEntry represents a cached JTI with metadata
type jtiEntry struct {
subject string
expiresAt time.Time
// uncounted marks an entry merged in past the capacity bound: it keeps
// rejecting replays but does not charge jtiCount, so merged excess can
// neither reject new exchanges nor be double-counted when swept.
uncounted bool
}
// OIDCProvider implements OpenID Connect authentication with JTI replay protection.
//
// Replay Protection:
// The provider validates the 'jti' (JWT ID) claim to prevent token replay attacks.
// Each JTI is stored in memory until the token expires. If a token with a duplicate
// JTI is presented, it is rejected with ErrProviderTokenReplayed.
//
// The JTI cache is bounded by JTICacheMaxSize and uses automatic TTL-based expiration.
// A token without a usable jti claim is tracked by the token's own fingerprint
// instead, so a repeated presentation is still detected as a replay.
//
// Thread Safety:
// All methods are safe for concurrent use. The JTI cache uses atomic LoadOrStore
// operations to prevent TOCTOU race conditions.
type OIDCProvider struct {
name string
config *OIDCConfig
@@ -44,6 +83,32 @@ type OIDCProvider struct {
jwksFetchedAt time.Time
resolvedJWKSUri string
discoveryFailed bool
// JTI replay protection fields. jtiStore and jtiCount are allocated once
// in NewOIDCProvider and survive re-initialization, so a config refresh
// cannot reopen already-consumed tokens for replay.
jtiEnabled atomic.Bool
jtiStore *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys
jtiCount *atomic.Int64 // entries in jtiStore; shared with the store
jtiMaxSize atomic.Int64
jtiLastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups
// jtiStoreMu guards the jtiStore/jtiCount pair: replay-state handoffs
// (AdoptReplayStateFrom, MergeReplayStateFrom) repoint them under the
// write lock while the consume path holds a read lock, so a consume can
// never land in a store the handoff already drained and abandoned.
// It is always acquired before jtiSweepMu, never while holding it.
jtiStoreMu sync.RWMutex
// jtiMu guards the lifecycle of the cleanup worker: jtiCleanupCancel and
// jtiCleanupDone. The worker itself only touches jtiStore/jtiCount, so
// holding jtiMu while waiting for it is safe.
jtiMu sync.Mutex
jtiCleanupCancel context.CancelFunc
jtiCleanupDone chan struct{} // Closed when cleanup goroutine exits
// jtiSweepMu serializes cache sweeps so at most one scan runs at a time.
jtiSweepMu sync.Mutex
}
// OIDCConfig holds OIDC provider configuration
@@ -107,6 +172,13 @@ type OIDCConfig struct {
// TLSInsecureSkipVerify controls whether to skip TLS verification.
// WARNING: Should only be used in development/testing environments. Never use in production.
TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty"`
// JTIReplayProtectionEnabled enables token replay protection via JTI tracking (default: true)
// Use pointer to distinguish between "not set" (nil = default true) and "explicitly false"
JTIReplayProtectionEnabled *bool `json:"jtiReplayProtectionEnabled,omitempty"`
// JTICacheMaxSize sets maximum number of JTI entries to cache (default: 10000)
JTICacheMaxSize int `json:"jtiCacheMaxSize,omitempty"`
}
// PrincipalTagsClaim is the AWS-defined namespace claim that carries
@@ -319,6 +391,8 @@ func NewOIDCProvider(name string) *OIDCProvider {
return &OIDCProvider{
name: name,
httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect},
jtiStore: &sync.Map{},
jtiCount: &atomic.Int64{},
}
}
@@ -429,10 +503,62 @@ func (p *OIDCProvider) Initialize(config interface{}) error {
CheckRedirect: refuseDowngradeRedirect,
}
// Configure JTI replay protection last, so a config failure above cannot
// leave a running worker behind. A previous worker is always stopped,
// whether the new configuration enables protection or disables it.
// The consumed-token store survives re-initialization: resetting it would
// let already-seen tokens be replayed against the refreshed provider.
jtiEnabled := true
if oidcConfig.JTIReplayProtectionEnabled != nil {
jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled
}
p.jtiMu.Lock()
p.stopJTICleanupWorkerLocked()
if jtiEnabled {
maxSize := int64(oidcConfig.JTICacheMaxSize)
if maxSize <= 0 {
maxSize = defaultJTICacheSize
}
// the bound must land before the flag: a consume that sees enabled
// with a zero bound would reject the first token it registers
p.jtiMaxSize.Store(maxSize)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
p.jtiCleanupCancel = cancel
p.jtiCleanupDone = done
p.jtiEnabled.Store(true)
p.jtiMu.Unlock()
go p.cleanupExpiredJTIs(ctx, done, defaultCleanupInterval)
glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", p.name, maxSize)
} else {
p.jtiEnabled.Store(false)
p.jtiMu.Unlock()
glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", p.name)
}
// For testing, we'll skip the actual OIDC client initialization
return nil
}
// stopJTICleanupWorkerLocked cancels the current cleanup goroutine and waits
// briefly for it to exit. Caller must hold jtiMu.
func (p *OIDCProvider) stopJTICleanupWorkerLocked() {
if p.jtiCleanupCancel == nil {
return
}
p.jtiCleanupCancel()
select {
case <-p.jtiCleanupDone:
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", p.name)
case <-time.After(2 * time.Second):
glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", p.name)
}
p.jtiCleanupCancel = nil
p.jtiCleanupDone = nil
}
// validateConfig validates the OIDC configuration
func (p *OIDCProvider) validateConfig(config *OIDCConfig) error {
if config.Issuer == "" {
@@ -479,6 +605,18 @@ func (p *OIDCProvider) allowedAudiences() []string {
// Authenticate authenticates a user with an OIDC token
func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
return p.authenticate(ctx, token, false)
}
// AuthenticateSingleUse authenticates and consumes the token so a second
// presentation is rejected as a replay. The web-identity exchange uses it,
// where a token buys a session exactly once; callers that validate the same
// token on every request use Authenticate.
func (p *OIDCProvider) AuthenticateSingleUse(ctx context.Context, token string) (*providers.ExternalIdentity, error) {
return p.authenticate(ctx, token, true)
}
func (p *OIDCProvider) authenticate(ctx context.Context, token string, singleUse bool) (*providers.ExternalIdentity, error) {
if !p.initialized {
return nil, fmt.Errorf("provider not initialized")
}
@@ -488,7 +626,7 @@ func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*provide
}
// Validate token and get claims
claims, err := p.ValidateToken(ctx, token)
claims, err := p.validateToken(ctx, token, singleUse)
if err != nil {
return nil, err
}
@@ -655,6 +793,16 @@ func (p *OIDCProvider) getUserInfoWithToken(ctx context.Context, userID, accessT
// ValidateToken validates an OIDC JWT token
func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*providers.TokenClaims, error) {
return p.validateToken(ctx, token, false)
}
// ValidateTokenSingleUse validates and consumes the token: a second
// presentation of it is rejected as a replay. See AuthenticateSingleUse.
func (p *OIDCProvider) ValidateTokenSingleUse(ctx context.Context, token string) (*providers.TokenClaims, error) {
return p.validateToken(ctx, token, true)
}
func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUse bool) (*providers.TokenClaims, error) {
if !p.initialized {
return nil, fmt.Errorf("provider not initialized")
}
@@ -787,6 +935,87 @@ func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*provid
tokenClaims.Claims[key] = value
}
// Extract and validate JTI claim for replay protection. The check runs
// only for single-use callers: a bearer token is meant to be presented on
// every request, so validating it must not consume it.
if singleUse && p.jtiEnabled.Load() {
jti, _ := claims["jti"].(string)
key := jti
if key == "" {
// Providers that issue no jti cannot be tracked by claim, so the
// signed input keys the entry: a repeated presentation of the same
// claims is a replay even when the signature bytes differ (e.g.
// ECDSA s/N-s malleability).
signingInput := token
if i := strings.LastIndexByte(token, '.'); i > 0 {
signingInput = token[:i]
}
sum := sha256.Sum256([]byte(signingInput))
key = tokenFingerprintPrefix + hex.EncodeToString(sum[:])
}
expiresAt := tokenClaims.ExpiresAt
if expiresAt.IsZero() {
// a cache slot is freed only by expiry, so a token without exp
// would hold it forever and eventually exhaust the bound
return nil, fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims)
}
expiresAt = expiresAt.Add(clockSkewTolerance)
if !time.Now().Before(expiresAt) {
return nil, fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired)
}
entry := &jtiEntry{
subject: subject,
expiresAt: expiresAt,
}
// The read lock pins the store/count pair for the whole test-and-set
// so a replay-state handoff cannot repoint them mid-check and strand
// this consume in an abandoned store.
p.jtiStoreMu.RLock()
store, count := p.jtiStore, p.jtiCount
// Atomic test-and-set detects replays before any capacity work runs,
// so a replayed token cannot trigger cleanup scans.
if _, loaded := store.LoadOrStore(key, entry); loaded {
p.jtiStoreMu.RUnlock()
glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)",
p.name, jtiLogKey(key))
return nil, fmt.Errorf("%w: jti %s has already been used",
providers.ErrProviderTokenReplayed, jtiLogKey(key))
}
// The count is incremented as part of the insert, so concurrent
// validations cannot all slip under the limit: whoever pushes the
// count over evicts their own entry and is rejected.
newCount := count.Add(1)
maxSize := p.jtiMaxSize.Load()
if newCount > maxSize {
// Over the bound: expired entries may still occupy slots, so
// reclaim them with a synchronized sweep before rejecting.
p.sweepJTIStore(store, count)
if count.Load() > maxSize {
if _, loaded := store.LoadAndDelete(key); loaded {
count.Add(-1)
}
p.jtiStoreMu.RUnlock()
glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+
"This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.",
p.name, maxSize)
return nil, fmt.Errorf("JTI cache capacity exceeded")
}
} else if newCount >= maxSize*9/10 {
glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup",
p.name, (newCount*100)/maxSize, newCount, maxSize)
go p.performEagerCleanup()
}
p.jtiStoreMu.RUnlock()
glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)",
p.name, jtiLogKey(key), expiresAt, newCount)
}
return tokenClaims, nil
}
@@ -1215,3 +1444,195 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p
return identity
}
// evictExpiredJTIs removes entries whose expiry has passed and returns how
// many it deleted. Caller must hold jtiSweepMu.
func (p *OIDCProvider) evictExpiredJTIs(now time.Time, store *sync.Map, count *atomic.Int64) int {
removed := 0
store.Range(func(key, value interface{}) bool {
expiresAt := value.(*jtiEntry).expiresAt
if !expiresAt.IsZero() && now.After(expiresAt) {
// LoadAndDelete reports whether this call actually removed the
// entry, so concurrent deletes cannot decrement the count twice.
if v, loaded := store.LoadAndDelete(key); loaded {
if !v.(*jtiEntry).uncounted {
count.Add(-1)
}
removed++
}
}
return true
})
return removed
}
// replayState returns the provider's current store/count pair.
func (p *OIDCProvider) replayState() (*sync.Map, *atomic.Int64) {
p.jtiStoreMu.RLock()
defer p.jtiStoreMu.RUnlock()
return p.jtiStore, p.jtiCount
}
// sweepJTIStore runs one serialized expiry scan over the given store. It only
// takes jtiSweepMu, so callers already holding jtiStoreMu may use it.
func (p *OIDCProvider) sweepJTIStore(store *sync.Map, count *atomic.Int64) {
p.jtiSweepMu.Lock()
defer p.jtiSweepMu.Unlock()
p.jtiLastSweep.Store(time.Now().UnixNano())
if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 {
glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)",
p.name, removed, count.Load())
}
}
// sweepExpiredJTIs runs a scan that is guaranteed to complete before the
// caller proceeds; the capacity-rejection path uses it so a fresh token is
// not refused while expired entries still hold slots.
func (p *OIDCProvider) sweepExpiredJTIs() {
store, count := p.replayState()
p.sweepJTIStore(store, count)
}
// performEagerCleanup performs an opportunistic cleanup of expired JTI
// entries when the cache approaches capacity. At most one sweep runs at a
// time and no more often than eagerCleanupMinInterval, so a burst of
// requests near the limit cannot each turn into a full-cache scan.
func (p *OIDCProvider) performEagerCleanup() {
if !p.jtiEnabled.Load() {
return
}
if time.Since(time.Unix(0, p.jtiLastSweep.Load())) < eagerCleanupMinInterval {
return
}
store, count := p.replayState()
if !p.jtiSweepMu.TryLock() {
return
}
defer p.jtiSweepMu.Unlock()
p.jtiLastSweep.Store(time.Now().UnixNano())
beforeCount := count.Load()
if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 {
glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)",
p.name, removed, beforeCount, count.Load())
} else {
glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+
"Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.",
p.name)
}
}
// cleanupExpiredJTIs periodically removes expired JTI entries from the cache.
// This prevents memory growth by removing JTIs after their tokens have expired.
// Runs in a background goroutine until ctx is cancelled; done is closed on exit.
func (p *OIDCProvider) cleanupExpiredJTIs(ctx context.Context, done chan struct{}, interval time.Duration) {
defer close(done)
ticker := time.NewTicker(interval)
defer ticker.Stop()
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", p.name, interval)
for {
select {
case <-ctx.Done():
glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", p.name)
return
case <-ticker.C:
p.sweepExpiredJTIs()
}
}
}
// Shutdown gracefully stops the JTI cleanup goroutine.
// Should be called when the provider is being shut down to prevent goroutine leaks.
func (p *OIDCProvider) Shutdown(ctx context.Context) error {
p.jtiMu.Lock()
cancel, done := p.jtiCleanupCancel, p.jtiCleanupDone
p.jtiMu.Unlock()
if cancel == nil {
return nil
}
cancel()
select {
case <-done:
glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", p.name)
case <-ctx.Done():
glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", p.name, ctx.Err())
return ctx.Err()
}
p.jtiMu.Lock()
if p.jtiCleanupDone == done {
p.jtiCleanupCancel = nil
p.jtiCleanupDone = nil
}
p.jtiMu.Unlock()
return nil
}
// AdoptReplayStateFrom takes over the consumed-token store of the provider it
// replaces, so tokens it already saw stay rejected across the rebuild. The
// caller must shut down the replaced provider; only one cleanup worker may
// sweep the shared store at a time. Call before the new provider serves
// validators; jtiSweepMu excludes its own cleanup worker during the swap.
func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) {
if old == nil || old == p {
return
}
p.jtiStoreMu.Lock()
p.jtiSweepMu.Lock()
p.jtiStore, p.jtiCount = old.jtiStore, old.jtiCount
p.jtiSweepMu.Unlock()
p.jtiStoreMu.Unlock()
}
// MergeReplayStateFrom hands this provider's store to another provider and
// then folds the other's consumed-token entries in, so rejections the other
// recorded keep holding when its records move to a shared provider. Repointing
// first means tokens the other still consumes — in-flight exchanges or ones
// routed to it before the provider map updates — land in the shared store
// instead of a store this merge already drained. Unlike AdoptReplayStateFrom
// it swaps nothing of its own, so it is safe on a provider that is already
// serving validators.
func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) {
if other == nil || other == p {
return
}
p.jtiStoreMu.RLock()
defer p.jtiStoreMu.RUnlock()
store, count := p.jtiStore, p.jtiCount
other.jtiStoreMu.Lock()
oldStore := other.jtiStore
if oldStore == store {
other.jtiStoreMu.Unlock()
return
}
other.jtiStore, other.jtiCount = store, count
other.jtiStoreMu.Unlock()
p.jtiSweepMu.Lock()
defer p.jtiSweepMu.Unlock()
now := time.Now()
oldStore.Range(func(key, value interface{}) bool {
entry := value.(*jtiEntry)
if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) {
return true
}
// A consumed token is never dropped to fit the bound: merged entries
// stay but do not charge the count, so a store filled past the bound
// by the merge cannot reject new exchanges while it drains by expiry.
store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true})
return true
})
}
// jtiLogKey redacts a token identifier for logs and error strings: a stable
// fingerprint that still lets operators correlate repeated presentations.
func jtiLogKey(jti string) string {
sum := sha256.Sum256([]byte(jti))
return hex.EncodeToString(sum[:8])
}
weed/iam/oidc/oidc_provider_test.go
+1080 -12
View File
File diff suppressed because it is too large. Load diff
weed/iam/providers/errors.go
+3
View File
@@ -19,4 +19,7 @@ var (
// ErrProviderMissingClaims indicates that required claims are missing from the token
ErrProviderMissingClaims = errors.New("provider: missing required claims")
// ErrProviderTokenReplayed is returned when a token with duplicate JTI is detected
ErrProviderTokenReplayed = errors.New("provider: token has been replayed")
)
weed/iam/sts/constants.go
+17 -15
View File
@@ -42,21 +42,23 @@ const (
// Configuration Field Names
const (
ConfigFieldFilerAddress = "filerAddress"
ConfigFieldBasePath = "basePath"
ConfigFieldIssuer = "issuer"
ConfigFieldClientID = "clientId"
ConfigFieldClientIDs = "clientIds"
ConfigFieldThumbprints = "thumbprints"
ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys"
ConfigFieldPolicyClaim = "policyClaim"
ConfigFieldClientSecret = "clientSecret"
ConfigFieldJWKSUri = "jwksUri"
ConfigFieldScopes = "scopes"
ConfigFieldUserInfoUri = "userInfoUri"
ConfigFieldRedirectUri = "redirectUri"
ConfigFieldTLSCACert = "tlsCaCert"
ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify"
ConfigFieldFilerAddress = "filerAddress"
ConfigFieldBasePath = "basePath"
ConfigFieldIssuer = "issuer"
ConfigFieldClientID = "clientId"
ConfigFieldClientIDs = "clientIds"
ConfigFieldThumbprints = "thumbprints"
ConfigFieldAllowedPrincipalTagKeys = "allowedPrincipalTagKeys"
ConfigFieldPolicyClaim = "policyClaim"
ConfigFieldClientSecret = "clientSecret"
ConfigFieldJWKSUri = "jwksUri"
ConfigFieldScopes = "scopes"
ConfigFieldUserInfoUri = "userInfoUri"
ConfigFieldRedirectUri = "redirectUri"
ConfigFieldTLSCACert = "tlsCaCert"
ConfigFieldTLSInsecureSkipVerify = "tlsInsecureSkipVerify"
ConfigFieldJTIReplayProtectionEnabled = "jtiReplayProtectionEnabled"
ConfigFieldJTICacheMaxSize = "jtiCacheMaxSize"
)
// Error Messages
weed/iam/sts/oidc_provider_lookup_test.go
+50 -2
View File
@@ -11,8 +11,8 @@ import (
// lookupOIDCProviderForAccount; the lookup never calls Authenticate.
type stubIdentityProvider struct{ name string }
func (s *stubIdentityProvider) Name() string { return s.name }
func (s *stubIdentityProvider) Initialize(interface{}) error { return nil }
func (s *stubIdentityProvider) Name() string { return s.name }
func (s *stubIdentityProvider) Initialize(interface{}) error { return nil }
func (s *stubIdentityProvider) Authenticate(context.Context, string) (*providers.ExternalIdentity, error) {
return nil, nil
}
@@ -103,3 +103,51 @@ func TestLookupOIDCProviderForAccountFallsBackToStatic(t *testing.T) {
t.Fatalf("expected static-config provider as last resort, got %v ok=%v", got, ok)
}
}
// shutdownTrackingProvider records Shutdown calls so tests can observe which
// replaced providers the swap released.
type shutdownTrackingProvider struct {
stubIdentityProvider
shutdownCalls int32
}
func (s *shutdownTrackingProvider) Shutdown(context.Context) error {
s.shutdownCalls++
return nil
}
// A provider dropped from the managed map owns a background JTI cleanup
// worker; replacing the map must shut it down, while providers that stay in
// the map keep running.
func TestSetIAMManagedOIDCProvidersShutsDownDroppedProviders(t *testing.T) {
dropped := &shutdownTrackingProvider{stubIdentityProvider{name: "dropped"}, 0}
kept := &shutdownTrackingProvider{stubIdentityProvider{name: "kept"}, 0}
added := &shutdownTrackingProvider{stubIdentityProvider{name: "added"}, 0}
s := &STSService{issuerToProvider: map[string]providers.IdentityProvider{}}
s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
"https://a.example.com": {{Provider: dropped}},
"https://b.example.com": {{Provider: kept}, {AccountID: "111111111111", Provider: kept}},
})
s.SetIAMManagedOIDCProviders(map[string][]ScopedOIDCProvider{
"https://b.example.com": {{Provider: kept}},
"https://c.example.com": {{Provider: added}},
})
if dropped.shutdownCalls != 1 {
t.Fatalf("dropped provider was shut down %d times, want 1", dropped.shutdownCalls)
}
if kept.shutdownCalls != 0 {
t.Fatalf("kept provider was shut down %d times, want 0", kept.shutdownCalls)
}
if added.shutdownCalls != 0 {
t.Fatalf("newly added provider was shut down %d times, want 0", added.shutdownCalls)
}
// Clearing the map releases the remaining providers.
s.SetIAMManagedOIDCProviders(nil)
if kept.shutdownCalls != 1 || added.shutdownCalls != 1 {
t.Fatalf("clearing the map shut down kept=%d added=%d, want 1 each", kept.shutdownCalls, added.shutdownCalls)
}
}
weed/iam/sts/provider_factory.go
+16
View File
@@ -1,6 +1,7 @@
package sts
import (
"encoding/json"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -154,6 +155,21 @@ func (f *ProviderFactory) convertToOIDCConfig(configMap map[string]interface{})
config.TLSInsecureSkipVerify = tlsInsecureSkipVerify
}
if enabled, ok := configMap[ConfigFieldJTIReplayProtectionEnabled].(bool); ok {
config.JTIReplayProtectionEnabled = &enabled
}
switch maxSize := configMap[ConfigFieldJTICacheMaxSize].(type) {
case float64:
config.JTICacheMaxSize = int(maxSize)
case int:
config.JTICacheMaxSize = maxSize
case json.Number:
if n, err := maxSize.Int64(); err == nil {
config.JTICacheMaxSize = int(n)
}
}
// Convert claims mapping
if claimsMapInterface, ok := configMap["claimsMapping"]; ok {
claimsMap, err := f.convertToStringMap(claimsMapInterface)
weed/iam/sts/provider_factory_jti_test.go
+31
View File
@@ -0,0 +1,31 @@
package sts
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertToOIDCConfigCarriesJTISettings(t *testing.T) {
factory := NewProviderFactory()
config, err := factory.convertToOIDCConfig(map[string]interface{}{
"issuer": "https://issuer.example",
"clientId": "client",
"jtiReplayProtectionEnabled": false,
"jtiCacheMaxSize": float64(42),
})
require.NoError(t, err)
require.NotNil(t, config.JTIReplayProtectionEnabled)
assert.False(t, *config.JTIReplayProtectionEnabled)
assert.Equal(t, 42, config.JTICacheMaxSize)
config, err = factory.convertToOIDCConfig(map[string]interface{}{
"issuer": "https://issuer.example",
"clientId": "client",
})
require.NoError(t, err)
assert.Nil(t, config.JTIReplayProtectionEnabled)
assert.Zero(t, config.JTICacheMaxSize)
}
weed/iam/sts/session_claims.go
+13
View File
@@ -100,6 +100,11 @@ type STSSessionClaims struct {
// guaranteed stable per OpenID Connect Core 1.0. Empty for non-federated
// session types.
ParentUser string `json:"puid,omitempty"`
// SourceAccessKeyId is the access key the caller authenticated with when
// a SeaweedFS service minted the session for it (AssumeRoleForPrincipal);
// empty for every other session type.
SourceAccessKeyId string `json:"src_ak,omitempty"`
}
// NewSTSSessionClaims creates new STS session claims with all required information
@@ -156,6 +161,8 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI
ProviderIssuer: c.ProviderIssuer,
RequestContext: c.RequestContext,
ParentUser: c.ParentUser,
// The access key the session was minted for, if recorded
SourceAccessKeyId: c.SourceAccessKeyId,
// Provide the Subject (sub) from registered claims
Subject: c.Subject,
Credentials: credentials,
@@ -250,6 +257,12 @@ func (c *STSSessionClaims) WithSessionName(sessionName string) *STSSessionClaims
return c
}
// WithSourceAccessKeyId records the access key the session was minted for.
func (c *STSSessionClaims) WithSourceAccessKeyId(accessKeyId string) *STSSessionClaims {
c.SourceAccessKeyId = accessKeyId
return c
}
// WithParentUser sets the stable per-identity hash for the session. See
// ComputeParentUser for the derivation rule.
func (c *STSSessionClaims) WithParentUser(parentUser string) *STSSessionClaims {
weed/iam/sts/source_access_key_test.go
+42
View File
@@ -0,0 +1,42 @@
package sts
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newSourceKeyTestSTS(t *testing.T) *STSService {
t.Helper()
s := NewSTSService()
require.NoError(t, s.Initialize(&STSConfig{
TokenDuration: FlexibleDuration{Duration: time.Hour},
MaxSessionLength: FlexibleDuration{Duration: 12 * time.Hour},
Issuer: "test-issuer",
SigningKey: []byte("test-signing-key-at-least-32-bytes-long-for-security"),
}))
return s
}
// A session minted for a caller's access key carries that key back out of its
// token; a session minted without one carries none.
func TestSessionCarriesItsSourceAccessKey(t *testing.T) {
s := newSourceKeyTestSTS(t)
resp, err := s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t-kabc", "", nil,
"iceberg-catalog", "alice", func(c *STSSessionClaims) { c.WithSourceAccessKeyId("AKIASOURCE") })
require.NoError(t, err)
info, err := s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken)
require.NoError(t, err)
assert.Equal(t, "AKIASOURCE", info.SourceAccessKeyId)
assert.Equal(t, "alice", info.ExternalUserId)
assert.Equal(t, "iceberg-catalog", info.IdentityProvider)
resp, err = s.issueSession("arn:aws:iam::role/IcebergTableAccess", "", "iceberg-alice-ns-t", "", nil, "iceberg-catalog", "alice")
require.NoError(t, err)
info, err = s.ValidateSessionToken(context.Background(), resp.Credentials.SessionToken)
require.NoError(t, err)
assert.Equal(t, "", info.SourceAccessKeyId)
}
weed/iam/sts/sts_service.go
+64 -8
View File
@@ -316,6 +316,10 @@ type SessionInfo struct {
// RoleId is the unique ID of the role the session was issued for; empty
// for sessions issued before role IDs were recorded.
RoleId string `json:"roleId,omitempty"`
// SourceAccessKeyId is the access key the session was minted for, when the
// minting service recorded it (AssumeRoleForPrincipal).
SourceAccessKeyId string `json:"sourceAccessKeyId,omitempty"`
}
// NewSTSService creates a new STS service
@@ -508,8 +512,35 @@ func (s *STSService) SetIAMManagedOIDCProviders(byIssuer map[string][]ScopedOIDC
}
}
s.iamManagedOIDCMu.Lock()
old := s.iamManagedOIDCByIssuer
s.iamManagedOIDCByIssuer = cp
s.iamManagedOIDCMu.Unlock()
// Providers dropped by the swap may own background workers (the JTI
// cleanup goroutine); stop them so a replaced provider does not leak.
kept := make(map[providers.IdentityProvider]struct{})
for _, scoped := range cp {
for _, sp := range scoped {
kept[sp.Provider] = struct{}{}
}
}
stopped := make(map[providers.IdentityProvider]struct{})
for _, scoped := range old {
for _, sp := range scoped {
if _, ok := kept[sp.Provider]; ok {
continue
}
if _, ok := stopped[sp.Provider]; ok {
continue
}
stopped[sp.Provider] = struct{}{}
if sd, ok := sp.Provider.(interface{ Shutdown(context.Context) error }); ok {
if err := sd.Shutdown(context.Background()); err != nil {
glog.Warningf("OIDC provider %q shutdown after replacement: %v", sp.Provider.Name(), err)
}
}
}
}
}
// lookupOIDCProviderForAccount returns the provider that should validate
@@ -583,7 +614,7 @@ func (s *STSService) AssumeRoleWithWebIdentity(ctx context.Context, request *Ass
// caller passed a legacy or claim-based ARN, in which case lookup falls
// back to a global (account-less) record only.
roleAccountID := utils.ParseRoleARN(request.RoleArn).AccountID
externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID)
externalIdentity, provider, err := s.validateWebIdentityToken(ctx, request.WebIdentityToken, roleAccountID, true)
if err != nil {
return nil, fmt.Errorf("failed to validate web identity token: %w", err)
}
@@ -780,7 +811,7 @@ func (s *STSService) AssumeRoleWithCredentials(ctx context.Context, request *Ass
// issueSession mints temporary credentials and the self-contained JWT that
// carries the whole session, shared by every assume-role entry point.
func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolicy string,
durationSeconds *int64, providerName, subject string) (*AssumeRoleResponse, error) {
durationSeconds *int64, providerName, subject string, options ...func(*STSSessionClaims)) (*AssumeRoleResponse, error) {
sessionDuration := s.CalculateSessionDuration(durationSeconds)
expiresAt := time.Now().Add(sessionDuration)
@@ -810,6 +841,9 @@ func (s *STSService) issueSession(roleArn, roleId, roleSessionName, sessionPolic
if sessionPolicy != "" {
sessionClaims.WithSessionPolicy(sessionPolicy)
}
for _, option := range options {
option(sessionClaims)
}
jwtToken, err := s.tokenGenerator.GenerateJWTWithClaims(sessionClaims)
if err != nil {
@@ -863,6 +897,10 @@ type AssumeRoleForPrincipalRequest struct {
// DurationSeconds requests a session lifetime.
DurationSeconds *int64
// SourceAccessKeyId optionally records the access key the caller
// authenticated with, so the session can be tied to that key.
SourceAccessKeyId string
}
// AssumeRoleForPrincipal issues session credentials for a caller that a
@@ -908,7 +946,8 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume
// Iceberg credential vending resolves no role definition, so its sessions
// carry no role ID and stay bound by name.
return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy,
request.DurationSeconds, request.ProviderName, request.Principal)
request.DurationSeconds, request.ProviderName, request.Principal,
func(c *STSSessionClaims) { c.WithSourceAccessKeyId(request.SourceAccessKeyId) })
}
// ValidateSessionToken validates a session token and returns session information
@@ -969,7 +1008,7 @@ func (s *STSService) validateAssumeRoleWithWebIdentityRequest(request *AssumeRol
// validateWebIdentityToken validates the web identity token with strict issuer-to-provider mapping
// SECURITY: JWT tokens with a specific issuer claim MUST only be validated by the provider for that issuer
// SECURITY: This method only accepts JWT tokens. Non-JWT authentication must use AssumeRoleWithCredentials with explicit ProviderName.
func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAccountID string, singleUse bool) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
// Try to extract issuer from JWT token for strict validation
issuer, err := s.extractIssuerFromJWT(token)
if err != nil {
@@ -992,8 +1031,10 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
return nil, nil, fmt.Errorf("no identity provider registered for issuer: %s", issuer)
}
// Authenticate with the correct provider for this issuer
identity, err := provider.Authenticate(ctx, token)
// Authenticate with the correct provider for this issuer. Only the STS
// exchange authenticates single-use: a bearer token is presented on every
// request, so consuming it here would reject all but the first one.
identity, err := authenticateOIDCProvider(ctx, provider, token, singleUse)
if err != nil {
// Map provider errors to STS errors using errors.Is() for robust error checking
// This eliminates fragile string matching and provides reliable error classification
@@ -1025,14 +1066,29 @@ func (s *STSService) validateWebIdentityToken(ctx context.Context, token, roleAc
// only, then static-config); call ValidateWebIdentityTokenForAccount when the assumed-role account
// is known.
func (s *STSService) ValidateWebIdentityToken(ctx context.Context, token string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
return s.validateWebIdentityToken(ctx, token, "")
return s.validateWebIdentityToken(ctx, token, "", false)
}
// ValidateWebIdentityTokenForAccount mirrors ValidateWebIdentityToken but
// scopes the IAM-managed provider lookup to roleAccountID. Pass "" for
// callers that don't yet know the account (e.g. claim-based mode).
func (s *STSService) ValidateWebIdentityTokenForAccount(ctx context.Context, token, roleAccountID string) (*providers.ExternalIdentity, providers.IdentityProvider, error) {
return s.validateWebIdentityToken(ctx, token, roleAccountID)
return s.validateWebIdentityToken(ctx, token, roleAccountID, false)
}
// authenticateOIDCProvider authenticates the token, consuming it for replay
// protection when the caller is the single-use STS exchange and the provider
// supports it. Other providers and non-exchange callers authenticate without
// consuming.
func authenticateOIDCProvider(ctx context.Context, provider providers.IdentityProvider, token string, singleUse bool) (*providers.ExternalIdentity, error) {
if singleUse {
if su, ok := provider.(interface {
AuthenticateSingleUse(context.Context, string) (*providers.ExternalIdentity, error)
}); ok {
return su.AuthenticateSingleUse(ctx, token)
}
}
return provider.Authenticate(ctx, token)
}
// extractIssuerFromJWT extracts the issuer (iss) claim from a JWT token without verification
weed/iamapi/iamapi_server.go
+8 -6
View File
@@ -42,10 +42,11 @@ type IamS3ApiConfigure struct {
}
type IamServerOption struct {
Masters map[string]pb.ServerAddress
Filers []pb.ServerAddress
Port int
GrpcDialOption grpc.DialOption
Masters map[string]pb.ServerAddress
Filers []pb.ServerAddress
Port int
GrpcDialOption grpc.DialOption
IamSelfServiceRequiresGrant bool
}
type IamApiServer struct {
@@ -87,8 +88,9 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
s3ApiConfigure = configure
s3Option := s3api.S3ApiServerOption{
Filers: option.Filers,
GrpcDialOption: option.GrpcDialOption,
Filers: option.Filers,
GrpcDialOption: option.GrpcDialOption,
IamSelfServiceRequiresGrant: option.IamSelfServiceRequiresGrant,
}
// Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified)
weed/mount/peer_registrar_test.go
+4 -4
View File
@@ -8,6 +8,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
)
// fakeFilerClient captures MountRegister/MountList calls and lets the test
@@ -17,22 +18,21 @@ type fakeFilerClient struct {
filer_pb.SeaweedFilerClient // embed for methods we don't need
mu sync.Mutex
registerCalls []filer_pb.MountRegisterRequest
registerCalls []*filer_pb.MountRegisterRequest
listResponse filer_pb.MountListResponse
}
func (f *fakeFilerClient) MountRegister(ctx context.Context, req *filer_pb.MountRegisterRequest, opts ...grpc.CallOption) (*filer_pb.MountRegisterResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.registerCalls = append(f.registerCalls, *req)
f.registerCalls = append(f.registerCalls, proto.Clone(req).(*filer_pb.MountRegisterRequest))
return &filer_pb.MountRegisterResponse{}, nil
}
func (f *fakeFilerClient) MountList(ctx context.Context, req *filer_pb.MountListRequest, opts ...grpc.CallOption) (*filer_pb.MountListResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
resp := f.listResponse // value copy
return &resp, nil
return proto.Clone(&f.listResponse).(*filer_pb.MountListResponse), nil
}
// fakeFilerFleet maps each addr to its own fakeFilerClient so a test can
weed/operation/assign_file_id.go
+3 -3
View File
@@ -157,11 +157,11 @@ func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialO
return ret, lastError
}
func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
func LookupJwt(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
WithMasterServerClient(ctx, false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
resp, grpcErr := masterClient.LookupVolume(ctx, &master_pb.LookupVolumeRequest{
VolumeOrFileIds: []string{fileId},
})
if grpcErr != nil {
weed/operation/chunked_file.go
+5 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
@@ -101,6 +102,9 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
if offset > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
if jwt != "" {
req.Header.Set("Authorization", security.BearerPrefix+jwt)
}
resp, err := util_http.Do(req)
if err != nil {
@@ -118,7 +122,7 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
case http.StatusPartialContent:
break
default:
return written, fmt.Errorf("Read chunk needle error: [%d] %s", resp.StatusCode, fileUrl)
return written, util_http.StatusError(fileUrl, resp)
}
return io.Copy(w, resp.Body)
weed/operation/submit.go
+37 -6
View File
@@ -10,6 +10,7 @@ import (
"path"
"strconv"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -50,6 +51,10 @@ type StoragePreference struct {
type GetMasterFn func(ctx context.Context) pb.ServerAddress
// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
// once; the batch assign above already fetched all their file ids.
const submitUploadWorkers = 4
func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
results := make([]SubmitResult, len(files))
var totalBytes int64
@@ -72,8 +77,11 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
}
ret, err := Assign(context.Background(), masterFn, grpcDialOption, ar)
if err != nil {
for index := range files {
for index, file := range files {
results[index].Error = err.Error()
if closer, ok := file.Reader.(io.Closer); ok {
closer.Close()
}
}
return results, err
}
@@ -87,14 +95,35 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
file.Server = ret.PublicUrl
}
file.Pref = pref
results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
if err != nil {
results[index].Error = err.Error()
}
results[index].Fid = file.Fid
results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
}
return results, nil
var wg sync.WaitGroup
var lastErrLock sync.Mutex
jobs := make(chan int)
for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
results[index].Size = size
if uploadErr != nil {
results[index].Error = uploadErr.Error()
lastErrLock.Lock()
err = uploadErr
lastErrLock.Unlock()
}
}
}()
}
for index := range files {
jobs <- index
}
close(jobs)
wg.Wait()
return results, err
}
func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
@@ -118,6 +147,8 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
fi, fiErr := fh.Stat()
if fiErr != nil {
glog.V(0).Info("Failed to stat file:", fullPathFilename)
fh.Close()
ret.Reader = nil
return ret, fiErr
}
ret.ModTime = fi.ModTime().UTC().Unix()
weed/operation/upload_chunked.go
+33 -4
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"hash"
"io"
"net/http"
"sort"
"sync"
"time"
@@ -45,6 +46,7 @@ type ChunkedUploadOption struct {
Cipher bool // encrypt data on volume servers
AssignFunc func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error)
UploadFunc func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) // Optional: for testing
LookupJwtFunc func(ctx context.Context, fileId string) security.EncodedJwt // Optional: renews the volume JWT for a cleanup delete
}
var chunkBufferPool = sync.Pool{
@@ -213,7 +215,7 @@ uploadLoop:
// failed write can still leave a copy behind. No chunk will name
// this fid whether we retry or give up here, and an unreferenced
// needle is not garbage vacuum can find, so drop it either way.
deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt)
deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt, opt)
if attempt == chunkAssignAttempts || !ShouldReassignUpload(uploadResultErr) || objectFailed() {
break
}
@@ -413,19 +415,46 @@ func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data
if firstErr != nil {
// A failed fan-out records no chunk, so roll back the copies that landed
// before the cancel rather than leaking them as orphans.
deleteChunkFromHolders(succeeded, fid, jwt)
deleteChunkFromHolders(succeeded, fid, jwt, opt)
return nil, firstErr
}
return first, nil
}
// cleanupJwtLookupTimeout bounds a cleanup-token renewal, covering the wait
// for a master plus the lookup RPC, so a master outage cannot stall the
// rollback of a failed fan-out.
const cleanupJwtLookupTimeout = 10 * time.Second
// deleteChunkFromHolders best-effort removes a needle from each holder it landed
// on, using type=replicate so the volume drops only its local copy. A failed
// delete falls back to vacuum reclaiming the orphan.
func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt) {
func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt, opt *ChunkedUploadOption) {
// An unresponsive holder can stall the upload past the write JWT's expiry,
// so renew the token whenever a holder rejects the delete as unauthorized.
// Each holder's delete is retried at most once, and a slow holder that
// outlives the renewed token can still trigger another renewal.
deleteJwt := jwt
for _, host := range hosts {
if err := util_http.Delete(fmt.Sprintf("http://%s/%s?type=replicate", host, fid), string(jwt)); err != nil {
deleteUrl := fmt.Sprintf("http://%s/%s?type=replicate", host, fid)
body, status, err := util_http.DeleteProxied(deleteUrl, string(deleteJwt))
if status == http.StatusUnauthorized && opt.LookupJwtFunc != nil {
lookupCtx, cancel := context.WithTimeout(context.Background(), cleanupJwtLookupTimeout)
token := opt.LookupJwtFunc(lookupCtx, fid)
cancel()
if token != "" {
deleteJwt = token
body, status, err = util_http.DeleteProxied(deleteUrl, string(deleteJwt))
}
}
if err != nil {
glog.Warningf("replica fan-out cleanup: delete %s from %s: %v", fid, host, err)
continue
}
switch status {
case http.StatusNotFound, http.StatusNoContent, http.StatusAccepted, http.StatusOK:
default:
glog.Warningf("replica fan-out cleanup: delete %s from %s: status %d: %s", fid, host, status, body)
}
}
}
weed/operation/upload_chunked_test.go
+89 -4
View File
@@ -26,13 +26,11 @@ func TestUploadReaderInChunksReturnsPartialResultsOnError(t *testing.T) {
testData := bytes.Repeat([]byte("test data for chunk upload failure testing"), 1000) // ~40KB
reader := bytes.NewReader(testData)
uploadAttempts := 0
var uploadAttempts int32
// Create a mock assign function that succeeds for first chunk, then fails
assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
uploadAttempts++
if uploadAttempts == 1 {
if atomic.AddInt32(&uploadAttempts, 1) == 1 {
// First chunk succeeds
return nil, &AssignResult{
Fid: "test-fid-1,1234",
@@ -679,3 +677,90 @@ func TestUploadReaderInChunksAppendsFsyncWhenAssigned(t *testing.T) {
t.Errorf("Expected the upload URL to carry ?fsync=true when the assignment says fsync, got %q", gotUploadUrl)
}
}
// A holder that stalls past the assignment JWT's expiry answers the rollback
// delete with 401; the cleanup renews the token instead of leaking the copy.
func TestDeleteChunkFromHoldersRenewsExpiredJwt(t *testing.T) {
var staleDeletes, freshDeletes int32
holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusCreated)
return
}
switch r.Header.Get("Authorization") {
case "Bearer fresh-jwt":
atomic.AddInt32(&freshDeletes, 1)
w.WriteHeader(http.StatusAccepted)
default:
atomic.AddInt32(&staleDeletes, 1)
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprint(w, `{"error":"wrong jwt"}`)
}
}))
defer holder.Close()
lookups := 0
deleteChunkFromHolders(
[]string{strings.TrimPrefix(holder.URL, "http://")},
"1,0a0b0c0d",
"stale-jwt",
&ChunkedUploadOption{
LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt {
lookups++
return "fresh-jwt"
},
},
)
if lookups != 1 {
t.Fatalf("expected one jwt renewal, got %d", lookups)
}
if atomic.LoadInt32(&freshDeletes) != 1 {
t.Fatalf("expected the delete retried with the fresh jwt, got %d fresh deletes", freshDeletes)
}
}
// A later holder that rejects the renewed token can trigger another renewal:
// the one-retry limit is per holder, not per cleanup pass.
func TestDeleteChunkFromHoldersRenewsPerHolder(t *testing.T) {
var acceptedDeletes int32
holder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusCreated)
return
}
if r.Header.Get("Authorization") == "Bearer jwt-2" {
atomic.AddInt32(&acceptedDeletes, 1)
w.WriteHeader(http.StatusAccepted)
return
}
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprint(w, `{"error":"wrong jwt"}`)
}))
defer holder.Close()
tokens := []security.EncodedJwt{"jwt-1", "jwt-2"}
lookups := 0
deleteChunkFromHolders(
[]string{
strings.TrimPrefix(holder.URL, "http://"),
strings.TrimPrefix(holder.URL, "http://"),
},
"1,0a0b0c0d",
"stale-jwt",
&ChunkedUploadOption{
LookupJwtFunc: func(ctx context.Context, fileId string) security.EncodedJwt {
token := tokens[lookups]
lookups++
return token
},
},
)
if lookups != 2 {
t.Fatalf("expected a renewal per rejecting holder, got %d", lookups)
}
if atomic.LoadInt32(&acceptedDeletes) != 1 {
t.Fatalf("expected the second holder's retried delete to land, got %d", acceptedDeletes)
}
}
weed/pb/filer_pb/filer_client.go
+10
View File
@@ -67,6 +67,16 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.Fu
// consumers would mistake for an entry.
const ListSnapshotTsNsTrailerKey = "sw-list-snapshot-ts-ns"
// TraverseModeHeaderKey is gRPC metadata on a TraverseBfsMetadata stream. As
// request metadata, TraverseModeSnapshot asks for the entries from one store
// snapshot (any order, entry_ts_ns left 0) instead of a breadth-first walk; as
// a response header it says what the filer did. An older filer sends none.
const (
TraverseModeHeaderKey = "sw-traverse-mode"
TraverseModeSnapshot = "snapshot"
TraverseModeBfs = "bfs"
)
type EachEntryFunction func(entry *Entry, isLast bool) error
func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (err error) {
Loaded 100 of 213 files, more files were not shown because too many files have changed in this diff. Show more