mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
Compare commits
117
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e003bae7d | ||
|
|
4ac3633f1c | ||
|
|
4a5dbe4c6f | ||
|
|
ba1bfa816c | ||
|
|
be8ebc2e04 | ||
|
|
bfbeed61a7 | ||
|
|
05427e8108 | ||
|
|
f7323f2b63 | ||
|
|
f5e091cf91 | ||
|
|
c9f3a02b94 | ||
|
|
6bdcd17ed4 | ||
|
|
729b872f01 | ||
|
|
972f85159e | ||
|
|
ed7324611c | ||
|
|
05a1463855 | ||
|
|
c1c95e127f | ||
|
|
668ca45029 | ||
|
|
dcbea0626f | ||
|
|
de4075660f | ||
|
|
652133a28a | ||
|
|
e012fca9a2 | ||
|
|
3159b8408c | ||
|
|
f88751c3ff | ||
|
|
2089abecf3 | ||
|
|
6d0a6e0935 | ||
|
|
9b94f00325 | ||
|
|
d2a0002869 | ||
|
|
c01e87689e | ||
|
|
7b4bdb77bb | ||
|
|
1fd5008d63 | ||
|
|
646549eee1 | ||
|
|
9cd183b04f | ||
|
|
23eef59417 | ||
|
|
04c7b12238 | ||
|
|
f5418845ce | ||
|
|
53bc62c773 | ||
|
|
1f5cfc7552 | ||
|
|
b1c315b837 | ||
|
|
aceb089bc8 | ||
|
|
c7d8b106d5 | ||
|
|
a3b2fd722d | ||
|
|
9f413ff2c8 | ||
|
|
e037a9c1ce | ||
|
|
3db54ee7e5 | ||
|
|
94de65b3b5 | ||
|
|
ebc77400ca | ||
|
|
ecbf7af1ca | ||
|
|
b64895ed0a | ||
|
|
72ac9e96b2 | ||
|
|
11af0e0c12 | ||
|
|
de5b3305a7 | ||
|
|
04e135c21e | ||
|
|
862913d199 | ||
|
|
8c6d09f498 | ||
|
|
eb1dca8088 | ||
|
|
deacd575ea | ||
|
|
194e6ede9f | ||
|
|
97f49571a1 | ||
|
|
91eaabf371 | ||
|
|
c91da3d1f8 | ||
|
|
c76fb36416 | ||
|
|
0d3f5a220c | ||
|
|
c4e1802e3e | ||
|
|
1d30b0de21 | ||
|
|
5681be241e | ||
|
|
8ba33fae71 | ||
|
|
d8a47042c5 | ||
|
|
dbb2fdb93b | ||
|
|
b74971b6d4 | ||
|
|
95f46c8fff | ||
|
|
386385101d | ||
|
|
6342876392 | ||
|
|
40182a68bc | ||
|
|
eed00c3737 | ||
|
|
b449301a4e | ||
|
|
9e5357782e | ||
|
|
f4e638fa1d | ||
|
|
1e31c6dbf5 | ||
|
|
0f0f3811bf | ||
|
|
53b234558e | ||
|
|
d9c2d99f8f | ||
|
|
e3769592a5 | ||
|
|
c3f4b3e0eb | ||
|
|
438cc8562f | ||
|
|
51ab451f37 | ||
|
|
34927705fa | ||
|
|
d5b4e1352e | ||
|
|
5ef99c1d8d | ||
|
|
8e15110b4f | ||
|
|
f13b763552 | ||
|
|
df30c883fe | ||
|
|
5a1b0abc20 | ||
|
|
b2dd93eac8 | ||
|
|
5f7fefadae | ||
|
|
c09009a74f | ||
|
|
c0509cf499 | ||
|
|
63806b7073 | ||
|
|
504de9b562 | ||
|
|
d80e905fc7 | ||
|
|
799fd10342 | ||
|
|
a91b333658 | ||
|
|
4f70ad015e | ||
|
|
ddd3b3666f | ||
|
|
cdb01f8150 | ||
|
|
b067266e6b | ||
|
|
b3a58506aa | ||
|
|
b42993d24a | ||
|
|
fd4603d657 | ||
|
|
53377bcb5d | ||
|
|
1167ca8cf5 | ||
|
|
512f898600 | ||
|
|
858f8cbad8 | ||
|
|
c819e3fbea | ||
|
|
6ad4b07598 | ||
|
|
65ecb6aa12 | ||
|
|
0a38738076 | ||
|
|
10f8b2d5b1 |
No files matched your search
@@ -37,7 +37,7 @@ jobs:
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
@@ -104,8 +104,51 @@ jobs:
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
|
||||
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
|
||||
test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0
|
||||
echo "S3 rollout settings render correctly"
|
||||
|
||||
echo "=== Testing S3 preStopSeconds ==="
|
||||
if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then
|
||||
echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1
|
||||
fi
|
||||
grep -q "must be at least 35" /tmp/s3-prestop-short.yaml
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 \
|
||||
--set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml
|
||||
grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml
|
||||
echo "S3 preStopSeconds renders correctly"
|
||||
|
||||
echo "=== Testing the S3 config checksum is stable across renders ==="
|
||||
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
|
||||
# Secret drew new keys for the checksum whenever lookup found nothing (on
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v6
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: actions/cache/restore@v6
|
||||
|
||||
@@ -168,9 +168,6 @@ jobs:
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-unit-tests-windows:
|
||||
name: Rust Unit Tests (Windows)
|
||||
runs-on: windows-latest
|
||||
@@ -208,9 +205,6 @@ jobs:
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
name: "S3 listing study tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
listing-study:
|
||||
name: s3-listing-study tools and strategies
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
defaults:
|
||||
run:
|
||||
working-directory: test/s3/listing_study
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Cache pinned listing tools
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: test/s3/listing_study/.work/tools
|
||||
key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }}
|
||||
|
||||
- name: Build weed and install the study's pinned tools
|
||||
run: make build-weed tools
|
||||
|
||||
- name: Run the listing study against SeaweedFS
|
||||
timeout-minutes: 15
|
||||
run: make test
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
tail -200 .work/weed.log || true
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-listing-study-results
|
||||
path: |
|
||||
test/s3/listing_study/.work/run/results.json
|
||||
test/s3/listing_study/.work/weed.log
|
||||
retention-days: 14
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.9', '3.11', '3.12']
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
@@ -68,7 +68,15 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Cache suite Maven dependencies
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: snowflake-s3compat-m2-${{ hashFiles('test/s3/snowflake/run.sh') }}
|
||||
restore-keys: |
|
||||
snowflake-s3compat-m2-
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
@@ -78,6 +86,8 @@ jobs:
|
||||
|
||||
- name: Run Snowflake S3Compat API tests
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
MAVEN_ARGS: -ntp -s ${{ github.workspace }}/test/s3/snowflake/maven-settings.xml
|
||||
run: |
|
||||
# Starts weed server, creates the buckets/objects the suite needs,
|
||||
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
|
||||
|
||||
@@ -684,6 +684,12 @@ jobs:
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
|
||||
- name: Cache Iceberg jars
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/seaweedfs-test-jars
|
||||
key: iceberg-jars-${{ hashFiles('test/s3tables/catalog_spark/setup_test.go') }}
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: "S3 Tigris Compatibility Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/tigris_compat/**'
|
||||
- '.github/workflows/s3-tigris-compat-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/tigris_compat/**'
|
||||
- '.github/workflows/s3-tigris-compat-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-tigris-compat-tests:
|
||||
name: s3-api-compat-tests (Tigris)
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run s3-api-compat-tests
|
||||
working-directory: test/s3/tigris_compat
|
||||
run: make test
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/tigris_compat
|
||||
run: tail -200 weed-test.log || true
|
||||
|
||||
- name: Upload report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tigris-compat-results
|
||||
path: test/s3/tigris_compat/results.json
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -410,7 +410,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -581,7 +581,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
@@ -810,7 +810,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
|
||||
@@ -6,12 +6,16 @@ on:
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -67,6 +67,8 @@ Download the latest binary from the [releases](https://github.com/seaweedfs/seaw
|
||||
curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
```
|
||||
|
||||
For SeaweedFS Enterprise binaries, pass `--enterprise` to the script: `... | bash -s -- --enterprise`.
|
||||
|
||||
Then start a ready-to-use S3 object store:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -271,6 +271,10 @@ curl "http://localhost:9333/vol/vacuum?garbageThreshold=0.4"
|
||||
Vacuuming makes a volume read-only, copies live needles to a new volume, and
|
||||
swaps it in.
|
||||
|
||||
The master runs one vacuum at a time. A request that arrives while the
|
||||
automatic vacuum or another request is running does nothing and gets
|
||||
`409 Conflict` with `{"error":"another vacuum is running, try again later"}`.
|
||||
|
||||
### Delete a collection
|
||||
|
||||
```bash
|
||||
|
||||
@@ -5,7 +5,7 @@ go 1.26.6
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.51.1
|
||||
cloud.google.com/go/storage v1.67.0
|
||||
cloud.google.com/go/storage v1.69.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -84,24 +84,24 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/v3 v3.7.1
|
||||
go.etcd.io/etcd/api/v3 v3.7.3
|
||||
go.etcd.io/etcd/client/v3 v3.7.3
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.47.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev v0.49.0
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e
|
||||
golang.org/x/image v0.46.0
|
||||
golang.org/x/net v0.62.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/tools v0.51.0 // indirect
|
||||
golang.org/x/tools v0.52.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.310.0
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 // indirect
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect
|
||||
google.golang.org/grpc v1.86.0-dev.0.20261027044109-d619b5d02089
|
||||
google.golang.org/protobuf v1.36.12
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
@@ -123,10 +123,10 @@ require (
|
||||
github.com/apache/iceberg-go v0.7.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
@@ -145,7 +145,7 @@ require (
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.32.0
|
||||
github.com/pkg/sftp v1.13.12
|
||||
github.com/rabbitmq/amqp091-go v1.15.0
|
||||
github.com/rabbitmq/amqp091-go v1.16.0
|
||||
github.com/rclone/rclone v1.75.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
@@ -162,6 +162,7 @@ require (
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3
|
||||
go.uber.org/atomic v1.12.0
|
||||
go.yaml.in/yaml/v3 v3.0.6
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
@@ -184,7 +185,7 @@ require (
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -247,7 +248,7 @@ require (
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.1 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.2 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
@@ -276,37 +277,35 @@ require (
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.1 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.6 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.3 // indirect
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a // indirect
|
||||
cloud.google.com/go/auth v0.24.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.10.0 // indirect
|
||||
cloud.google.com/go/iam v1.14.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.30.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.31.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.24.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
|
||||
@@ -320,22 +319,22 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6
|
||||
github.com/aws/smithy-go v1.28.8
|
||||
github.com/aws/smithy-go v1.28.9
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
@@ -370,14 +369,14 @@ require (
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.8 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.9 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.2 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.3 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
github.com/goccy/go-json v0.11.2 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
@@ -387,7 +386,7 @@ require (
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
@@ -419,8 +418,8 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.49.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.15 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -434,7 +433,7 @@ require (
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.29
|
||||
github.com/pierrec/lz4/v4 v4.1.33
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
@@ -456,7 +455,7 @@ require (
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
@@ -481,9 +480,9 @@ require (
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
|
||||
go.opentelemetry.io/otel v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
|
||||
@@ -493,7 +492,7 @@ require (
|
||||
go.uber.org/zap v1.28.0 // indirect
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/time v0.16.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
|
||||
@@ -4,8 +4,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
|
||||
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a h1:jW1ZLFy9oa5ai8LOI40z6s+DL1cO/L1KSm9hVGnibn8=
|
||||
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
|
||||
cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs=
|
||||
cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk=
|
||||
cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY=
|
||||
@@ -18,14 +18,14 @@ cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210Hl
|
||||
cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w=
|
||||
cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI=
|
||||
cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0=
|
||||
cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY=
|
||||
cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM=
|
||||
cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY=
|
||||
cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU=
|
||||
cloud.google.com/go/pubsub v1.51.1 h1:R3G1wCOxBO7jRpL8x2pdZMv1GAJDF6ax/m2zPOtvTNE=
|
||||
cloud.google.com/go/pubsub v1.51.1/go.mod h1:y2T0IKtW1iWwVvazYaRpqOAFO4gy2+O7dTDt9TWY/5U=
|
||||
cloud.google.com/go/pubsub/v2 v2.7.0 h1:MFrBTZZa6PDWZzCi4NJRsHKMm2w0a4oAaYNqwjgbQTE=
|
||||
cloud.google.com/go/pubsub/v2 v2.7.0/go.mod h1:JaFvWNVRk3Knoil/4M1ECeLOaI9D8drbmJWypQlK5aM=
|
||||
cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8=
|
||||
cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
|
||||
cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM=
|
||||
cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g=
|
||||
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
|
||||
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
|
||||
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
@@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 h1:zoPQ7KXGgzqeUmNWl+b6IwVAqsb5lROEJhqAmqXQtm0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3/go.mod h1:ei/sDx/dwDIUtZE3PQP1u8+qrA+CRy4q7qW+A2MbgB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4=
|
||||
@@ -60,8 +60,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
|
||||
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI=
|
||||
@@ -75,14 +75,12 @@ github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjw
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194/go.mod h1:rl0WumSN9gSo775DgvQv+wMQ8rlb0ES/1hU5jkMtLXg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 h1:rmzbFS2CkHq0y2L5LkMVWm+QXjjNEXxjmhrJqga/QGk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0 h1:eSgSuKdcpY7RTN1oXSzC6S39rPBImn03ubLMsWK4RRg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 h1:fxLusCG+GlGY1SM7BYAAiiSFhzKUs5zqs0P8knDqeFU=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1/go.mod h1:yO+OQpByKDLTvpEcsFFexgzpeR8eRfCFWAYzxkAu4bk=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
@@ -173,50 +171,50 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
|
||||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 h1:fD02zeo2Kc4hlFUvGX3FfA+gGOwd4BBcYgPDHDEzx9Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0/go.mod h1:/lB2ILjoT5/q0zSwtHnrJPjXBvjalnEIl0IgSEdvHec=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 h1:FghdY9ebeZf9iIan9J0xfBEID8arYibNBeZ1XHvs4oA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0/go.mod h1:wu7uOJ/2nAdenjvhHikeF6j0al99kfPVG6ZDU4e76OY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 h1:4EezYQsIY0xPbD6u7TbQLfnpicSMQgA8n9paomVV4Zo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5/go.mod h1:Yr4VqXnyoyjWnOWJck8BOidkJtleQ1c8ypgd2n0epxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 h1:D+Zmdt3T3P770MrqTmZvalbj6ADO7xA7n+qnNakGvrI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4/go.mod h1:dUYH3ia3qEO5ogaWDLp2tjtb4u4Y9xRbae7tOr1ClIA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
|
||||
github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
|
||||
github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
|
||||
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -509,12 +507,12 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I=
|
||||
github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w=
|
||||
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
|
||||
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
|
||||
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
|
||||
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
|
||||
github.com/go-openapi/errors v0.22.9 h1:HI9+SyVYiRzyeBQGv0CbWbQa+u2S0nny7G7AhY+UMiM=
|
||||
github.com/go-openapi/errors v0.22.9/go.mod h1:R73q66sXP2smzAGgFOI4nqtusRK2xN5pYCjsij+KX58=
|
||||
github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ=
|
||||
github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E=
|
||||
github.com/go-openapi/testify/v2 v2.8.0 h1:19QDx5b57p8KjO2E4sgs/woL4Akp+FGLRlVm2xn0Rho=
|
||||
github.com/go-openapi/testify/v2 v2.8.0/go.mod h1:4gUN8jC+RqOE2qjW/vDmfyTvl5BCht4qL7oca0odrak=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -546,8 +544,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
|
||||
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
|
||||
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
||||
@@ -563,8 +561,8 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFig
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
|
||||
@@ -649,8 +647,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@@ -915,10 +913,10 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.15 h1:StSf9TINInaZtr4oww2+kXmfwa9SkN//g/LwS19/UJ0=
|
||||
github.com/nats-io/nats-server/v2 v2.11.15/go.mod h1:zwhv8Y0PE3KHyKgznJc/9Xoai638SaJd83zzJ5GJn74=
|
||||
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
|
||||
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
|
||||
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
|
||||
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
@@ -929,8 +927,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
|
||||
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
|
||||
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec=
|
||||
github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
|
||||
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -979,8 +977,8 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7c
|
||||
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
|
||||
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014=
|
||||
github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c=
|
||||
github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
|
||||
github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
|
||||
github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
|
||||
@@ -1051,8 +1049,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw=
|
||||
github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
|
||||
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSwiwT7XLAVc=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=
|
||||
github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=
|
||||
@@ -1155,8 +1153,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2 h1:jUEsvCMD6fH25J8K/w3q/XnIx8W1lb8+YLaEEHIjHmc=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.2/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
|
||||
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
@@ -1351,36 +1349,36 @@ go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3 h1:2tumepXe+wi7Wab64uUO71K7ISQyUWLGC1DJbbc59sY=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3/go.mod h1:HOR9Qn4gb/1/hVAmYprmlLpvTe6ZgfgNS9Z/+Ty65po=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3 h1:5zMwNu3ycrzCeA5NwOfQbxFRhqq974chq1DrYMua4QI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3/go.mod h1:/OhzqYrF0U7d1NOk55VMyUQGbYD1WoIe4bJwxb47wOk=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3 h1:UrXcetgkuSEocaBlQlulQRJlXLEnUAHuBRbHh0lue1Q=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3/go.mod h1:DcPxpqcpV8WIaRi/2waPCx0/Uq8rQUweDJaDYZk1gEE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 h1:MCcYL7J6Vt/X0kjqbMZkekCmwsurbQRbL69vkiye2lk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0/go.mod h1:3jnStNwSufK+f5ktjL4EPcwtig4rtd81NS70lqHuXl8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
|
||||
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
|
||||
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0 h1:LaLVGJtIzHcvHjVs2OftmKcc+M9mdOqS8vZ1fBD1Yes=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0/go.mod h1:vF6H5sNDuQBJWgSX00iZgbIqk2MZZoVMH0ar5t5bOpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
|
||||
@@ -1400,8 +1398,8 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syod
|
||||
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
|
||||
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.opentelemetry.io/proto/otlp v1.11.1 h1:SCrPqH9NE5CmBrOqpjNCiT0X5mGwMyGERXdogfR7Yjw=
|
||||
go.opentelemetry.io/proto/otlp v1.11.1/go.mod h1:wq8W4aSf6bOy8RZyESOeQFr9Pu7wydWX/TY4sgvrTPg=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
@@ -1427,10 +1425,10 @@ go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA=
|
||||
gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0=
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0 h1:cTmJnWOmhb3dYRzX7PvNkq2m5mQdBLFFxZbl/21pfjk=
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0/go.mod h1:hbEEsNt6mGQoonGQi39VOjHfx2f13DOmWouh4hGsfrU=
|
||||
gocloud.dev v0.49.0 h1:02y4IBHX12xiCBK3Nv80IzuCmZXLsOfgpKwMEKzR9O4=
|
||||
gocloud.dev v0.49.0/go.mod h1:uN+hz2x8ieyW5RjeMetvmpV/nRqFOr7DF0CKVY0Owp8=
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0 h1:Jk1bywJ6LvqvBbxOQ0qs0q4aa+ORgg1FzjhTy+5O4cI=
|
||||
gocloud.dev/pubsub/natspubsub v0.49.0/go.mod h1:bmtxptoOPBNAgvTW9tWzoZvw9vzunDoNk6l4cHGd6nc=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0 h1:iDSspbLlDbPL2avESbDfMNBK4r3B6H04Ejx+/hqh9JE=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.47.0/go.mod h1:2tSZnGnGLPw2ayW4pVaUmnA1LCjP7rekpXPkaBUZt0o=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
@@ -1451,8 +1449,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e h1:HgI6W1OolKK6tcV74is+rl9SMT0njsHlp0TbQiGXI/A=
|
||||
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e/go.mod h1:DBoxwWhjaW8DJJw/YeM4zpeOKMC1FMXSblsYy+1mFSw=
|
||||
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
||||
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -1608,8 +1606,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1631,10 +1629,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 h1:CXq2XEi5aEyLjjPZIwqrmQH8ZY4ctUwK7uyAx3mjX/U=
|
||||
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941/go.mod h1:3gK0ehnhP0V7/x3ypGoLyT3d0n6TSi/EcsphIjHrcz4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 h1:DS/w3N8rsfrwxwgGSj6WhMsPo7aNQGmDx74J81zd0NQ=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941/go.mod h1:AuGAdi9KnXss7xbLxIKhCvk+Ro/SxpET/iS45DNYN+M=
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg=
|
||||
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
||||
+65
-26
@@ -7,12 +7,14 @@
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
# curl -fsSL ... | bash -s -- --enterprise --component all
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
# --enterprise Install SeaweedFS Enterprise binaries (from seaweedfs/artifactory)
|
||||
# --help Show this help message
|
||||
|
||||
set -euo pipefail
|
||||
@@ -24,6 +26,10 @@ LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
WORKER_INSTALLED=false
|
||||
VECTOR_INSTALLED=false
|
||||
VOLUME_INSTALLED=false
|
||||
ENTERPRISE=false
|
||||
# Enterprise assets carry this after the binary name, e.g. weed-vector-enterprise_linux_amd64.tar.gz.
|
||||
EDITION=""
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
@@ -53,6 +59,7 @@ while [ $# -gt 0 ]; do
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--large-disk) LARGE_DISK=true; shift ;;
|
||||
--dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--enterprise) ENTERPRISE=true; REPO="seaweedfs/artifactory"; EDITION="-enterprise"; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) error "Unknown option: $1. Use --help for usage." ;;
|
||||
esac
|
||||
@@ -81,15 +88,22 @@ detect_platform() {
|
||||
}
|
||||
|
||||
# Get latest release tag from GitHub API
|
||||
# The Enterprise repo also releases other products (vfs-*, previews), which can
|
||||
# be its latest release, so take the newest plain version tag like 4.53.
|
||||
get_latest_version() {
|
||||
local url="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
url="https://api.github.com/repos/${REPO}/releases?per_page=30"
|
||||
fi
|
||||
local body
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
body="$(curl -fsSL "$url")"
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
body="$(wget -qO- "$url")"
|
||||
else
|
||||
error "Neither curl nor wget found. Please install one."
|
||||
fi
|
||||
echo "$body" | grep '"tag_name"' | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' | grep -E '^[0-9]+(\.[0-9]+)+$' | head -1 || true
|
||||
}
|
||||
|
||||
# Download a file
|
||||
@@ -110,17 +124,22 @@ go_asset_name() {
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
suffix="${suffix}_large_disk"
|
||||
fi
|
||||
echo "${suffix}.tar.gz"
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
suffix="weed-enterprise-${suffix}"
|
||||
fi
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "${suffix}.zip"
|
||||
else
|
||||
echo "${suffix}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build Rust volume server asset name
|
||||
rust_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local prefix="weed-volume"
|
||||
local prefix="weed-volume${EDITION}"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
prefix="weed-volume_large_disk"
|
||||
else
|
||||
prefix="weed-volume"
|
||||
prefix="${prefix}_large_disk"
|
||||
fi
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$os" = "windows" ]; then
|
||||
@@ -130,6 +149,19 @@ rust_asset_name() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Enterprise publishes the Rust volume server for linux only.
|
||||
rust_volume_platform_supported() {
|
||||
case "$2" in
|
||||
amd64|arm64) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$1" in
|
||||
linux) return 0 ;;
|
||||
darwin|windows) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops
|
||||
# the installer on anything else: a rate limit or a network blip must not read
|
||||
# as "this release predates the binary" and quietly skip it.
|
||||
@@ -154,22 +186,23 @@ asset_exists() {
|
||||
# maintains tables through the namespace and never opens a volume file.
|
||||
worker_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
echo "weed-worker_${os}_${arch}.tar.gz"
|
||||
echo "weed-worker${EDITION}_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
# The vector engine behind the S3 Vectors API, built like the worker.
|
||||
vector_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "weed-vector_${os}_${arch}.zip"
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.zip"
|
||||
else
|
||||
echo "weed-vector_${os}_${arch}.tar.gz"
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
vector_platform_supported() {
|
||||
case "$1/$2" in
|
||||
linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64) return 0 ;;
|
||||
linux/amd64|linux/arm64) return 0 ;;
|
||||
darwin/amd64|darwin/arm64|windows/amd64) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -189,7 +222,10 @@ install_component() {
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
case "$asset_name" in
|
||||
*.zip) unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" ;;
|
||||
*) tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" ;;
|
||||
esac
|
||||
|
||||
# The Go release action puts the binary inside a directory
|
||||
local weed_bin
|
||||
@@ -202,20 +238,16 @@ install_component() {
|
||||
fi
|
||||
|
||||
chmod +x "$weed_bin"
|
||||
install_binary "$weed_bin" "weed"
|
||||
ok "Installed weed to ${INSTALL_DIR}/weed"
|
||||
local dest_name="weed"
|
||||
if [ "$os" = "windows" ]; then
|
||||
dest_name="weed.exe"
|
||||
fi
|
||||
install_binary "$weed_bin" "$dest_name"
|
||||
ok "Installed weed to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
volume-rust)
|
||||
# Check platform support for Rust volume server
|
||||
case "$os" in
|
||||
linux|darwin|windows) ;;
|
||||
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
rust_volume_platform_supported "$os" "$arch" || error "The Rust volume server is not available for ${os}/${arch}"
|
||||
|
||||
asset_name="$(rust_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
@@ -247,6 +279,7 @@ install_component() {
|
||||
dest_name="weed-volume.exe"
|
||||
fi
|
||||
install_binary "$rust_bin" "$dest_name"
|
||||
VOLUME_INSTALLED=true
|
||||
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
@@ -359,7 +392,11 @@ main() {
|
||||
case "$COMPONENT" in
|
||||
all)
|
||||
install_component "weed" "$os" "$arch"
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
if rust_volume_platform_supported "$os" "$arch"; then
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
else
|
||||
warn "Skipping the Rust volume server: no build for ${os}/${arch}"
|
||||
fi
|
||||
# The worker is published for linux amd64/arm64 only, and only by
|
||||
# releases new enough to carry it; skip either case rather than fail
|
||||
# an install that has already put two binaries in place.
|
||||
@@ -388,7 +425,7 @@ main() {
|
||||
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed: ${INSTALL_DIR}/weed"
|
||||
fi
|
||||
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
info " weed-volume: ${INSTALL_DIR}/weed-volume"
|
||||
fi
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
@@ -401,7 +438,9 @@ main() {
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
fi
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
|
||||
fi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.53"
|
||||
appVersion: "4.54"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.53.1
|
||||
version: 4.54.0
|
||||
@@ -569,6 +569,8 @@ s3:
|
||||
|
||||
With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
|
||||
|
||||
Outside all-in-one mode, the volume Ingress and the default volume route target the `client-volume` Service, which lists only Ready volume pods. The `volume` Service also publishes pods that are not Ready yet, so their DNS records can exist before readiness. An explicit `backendRefs` entry that names the `volume` Service can therefore reach a volume server before it is Ready. Name the `client-volume` Service instead, with `volume.port`, and copy its `metadata.name` from `helm template` because long release names are truncated.
|
||||
|
||||
## Network Policies
|
||||
|
||||
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
|
||||
@@ -718,7 +720,7 @@ The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverri
|
||||
|
||||
`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `<fullname>-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets.
|
||||
|
||||
The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
|
||||
The default `vector.metricsPort: 0` keeps TCP probes. Set `vector.metricsPort: 9329` to serve `/metrics`, `/health`, and `/ready` from the engine; readiness then checks master registration. Images older than 4.54 do not accept the metrics flag, so leave metrics off when pinning `image.tag` or `vector.imageOverride` to one. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
|
||||
|
||||
|
||||
To provision vector buckets during installation and upgrades, set:
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
|
||||
{{- if kindIs "invalid" $s3PreStopSeconds }}
|
||||
{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }}
|
||||
{{- end }}
|
||||
{{- $s3PreStopSeconds = $s3PreStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -64,7 +72,7 @@ spec:
|
||||
{{- tpl .Values.allInOne.tolerations . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.allInOne.priorityClassName }}
|
||||
priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
|
||||
@@ -260,6 +268,9 @@ spec:
|
||||
{{- if $auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
-sftp \
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.filer.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
@@ -69,7 +73,7 @@ spec:
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
|
||||
terminationGracePeriodSeconds: 60
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
{{- if .Values.filer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
@@ -278,6 +282,9 @@ spec:
|
||||
{{- if .Values.filer.s3.auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ include "seaweedfs.masterServerArg" . }} \
|
||||
{{- range .Values.filer.extraArgs }}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.s3.enabled }}
|
||||
{{- $preStopSeconds := .Values.s3.preStopSeconds | int }}
|
||||
{{- $minGracePeriod := add $preStopSeconds 15 }}
|
||||
{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }}
|
||||
{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -154,6 +159,9 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $preStopSeconds 0 }}
|
||||
-preStopSeconds={{ $preStopSeconds }} \
|
||||
{{- end }}
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \
|
||||
{{- if .Values.s3.icebergPort }}
|
||||
-port.iceberg={{ .Values.s3.icebergPort }} \
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.vector.enabled }}
|
||||
{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }}
|
||||
{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $volumeEnabled .Values.volume.ingress.enabled }}
|
||||
{{- /* Determine service name based on deployment mode */}}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{{- /* Ready-only backend for the volume ingress and HTTPRoute; the volume Service publishes not-ready pods for DNS.
|
||||
Not named volume-client: every volume-<key> name belongs to a group under .Values.volumes. */}}
|
||||
{{- if and .Values.volume.enabled (not .Values.allInOne.enabled) (or .Values.volume.ingress.enabled .Values.volume.httpRoute.enabled) }}
|
||||
{{- $name := include "seaweedfs.componentName" (list . "client-volume") }}
|
||||
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- range $vname, $volume := $volumes }}
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
|
||||
{{- if and $volume.enabled (eq (include "seaweedfs.componentName" (list $ $volumeName)) $name) }}
|
||||
{{- fail (printf "The volume Ingress/HTTPRoute Service %q has the same name as the %s Service after truncation. Shorten fullnameOverride or the release name." $name $volumeName) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
|
||||
{{- if .Values.volume.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.volume.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- /* No metrics port, so the volume ServiceMonitor does not scrape each pod twice */}}
|
||||
ports:
|
||||
- name: "swfs-volume"
|
||||
port: {{ .Values.volume.port }}
|
||||
targetPort: {{ .Values.volume.port }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
{{- end }}
|
||||
@@ -23,6 +23,9 @@ metadata:
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
{{- /* The master hands out ${POD_NAME}.<this service> as soon as a volume server heartbeats, before it is Ready.
|
||||
Ingress and HTTPRoute use the client-volume Service instead, which publishes only Ready pods. */}}
|
||||
publishNotReadyAddresses: true
|
||||
internalTrafficPolicy: {{ $volume.internalTrafficPolicy | default "Cluster" }}
|
||||
ports:
|
||||
- name: "swfs-volume"
|
||||
|
||||
@@ -136,13 +136,11 @@ func TestVectorNetworkPolicyOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorMetricsImageCompatibility(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
func TestVectorMetricsDefaultImage(t *testing.T) {
|
||||
output, _ := render(t, "vector.enabled=true", "vector.metricsPort=9329")
|
||||
for _, want := range []string{"--metrics-port=9329", "path: /ready", "path: /health"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("default image missing %q", want)
|
||||
}
|
||||
}
|
||||
out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") {
|
||||
t.Fatalf("incompatible image accepted: %v %s", err, out)
|
||||
}
|
||||
render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test")
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findDocument(documents []map[string]interface{}, kind, name string) map[string]interface{} {
|
||||
for _, document := range documents {
|
||||
if document["kind"] == kind && document["metadata"].(map[string]interface{})["name"] == name {
|
||||
return document
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestVolumeServicePublishesNotReadyPods(t *testing.T) {
|
||||
_, documents := render(t)
|
||||
service := findDocument(documents, "Service", "vector-test-seaweedfs-volume")
|
||||
if service == nil {
|
||||
t.Fatal("no volume service")
|
||||
}
|
||||
if service["spec"].(map[string]interface{})["publishNotReadyAddresses"] != true {
|
||||
t.Fatal("volume pod DNS names must resolve before the pods are Ready")
|
||||
}
|
||||
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
|
||||
t.Fatal("client-volume service rendered without ingress or httpRoute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeIngressUsesReadyOnlyService(t *testing.T) {
|
||||
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volume.metricsPort=9327")
|
||||
client := findDocument(documents, "Service", "vector-test-seaweedfs-client-volume")
|
||||
if client == nil {
|
||||
t.Fatal("no client-volume service")
|
||||
}
|
||||
spec := client["spec"].(map[string]interface{})
|
||||
if _, found := spec["publishNotReadyAddresses"]; found {
|
||||
t.Fatal("client-volume must publish only Ready pods")
|
||||
}
|
||||
for _, port := range spec["ports"].([]interface{}) {
|
||||
if port.(map[string]interface{})["name"] == "metrics" {
|
||||
t.Fatal("client-volume must not expose metrics, or the ServiceMonitor scrapes each pod twice")
|
||||
}
|
||||
}
|
||||
|
||||
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
|
||||
if ingress == nil {
|
||||
t.Fatal("no volume ingress")
|
||||
}
|
||||
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
|
||||
t.Fatalf("volume ingress backend is %v", backend)
|
||||
}
|
||||
|
||||
route := findDocument(documents, "HTTPRoute", "vector-test-seaweedfs-volume")
|
||||
if route == nil {
|
||||
t.Fatal("no volume httpRoute")
|
||||
}
|
||||
rule := route["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := rule["backendRefs"].([]interface{})[0].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
|
||||
t.Fatalf("volume httpRoute backend is %v", backend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeRoutingServiceDoesNotCollideWithVolumeGroups(t *testing.T) {
|
||||
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volumes.client.replicas=1")
|
||||
seen := map[string]bool{}
|
||||
for _, document := range documents {
|
||||
key := fmt.Sprint(document["kind"], "/", document["metadata"].(map[string]interface{})["name"])
|
||||
if seen[key] {
|
||||
t.Fatalf("%s rendered twice", key)
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
if !seen["Service/vector-test-seaweedfs-volume-client"] || !seen["Service/vector-test-seaweedfs-client-volume"] {
|
||||
t.Fatal("volumes.client group Service and the routing Service must both render")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeRoutingServiceRejectsTruncatedCollision(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
fullname := strings.Repeat("a", 49) + "-client"
|
||||
output, err := exec.Command("helm", "template", "test", "..", "--set", "fullnameOverride="+fullname, "--set", "volume.ingress.enabled=true").CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(output), "same name as the volume Service") {
|
||||
t.Fatalf("colliding routing Service accepted: %v %s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeIngressAllInOne(t *testing.T) {
|
||||
_, documents := render(t, "allInOne.enabled=true", "volume.ingress.enabled=true")
|
||||
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
|
||||
t.Fatal("client-volume service rendered in all-in-one mode")
|
||||
}
|
||||
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
|
||||
if ingress == nil {
|
||||
t.Fatal("no volume ingress")
|
||||
}
|
||||
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-all-in-one" {
|
||||
t.Fatalf("all-in-one volume ingress backend is %v", backend)
|
||||
}
|
||||
}
|
||||
@@ -1070,6 +1070,10 @@ filer:
|
||||
httpsPort: 0
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503 (see s3.preStopSeconds). Added to the filer pod's 60s
|
||||
# termination grace period.
|
||||
preStopSeconds: 0
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
# set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
@@ -1118,8 +1122,17 @@ s3:
|
||||
# maxUnavailable: 0
|
||||
# maxSurge: 25%
|
||||
updateStrategy: {}
|
||||
# Time the s3 pod is given to shut down, including any preStop hook
|
||||
# Time the s3 pod is given to shut down, including any preStop hook.
|
||||
# With preStopSeconds set, it must be at least preStopSeconds + 15 (the
|
||||
# in-flight request drain), or the chart refuses to render. A lifecycle
|
||||
# preStop hook runs before SIGTERM, so add its duration on top; the chart
|
||||
# cannot tell how long an exec hook takes. preStopSeconds replaces a
|
||||
# preStop sleep hook.
|
||||
terminationGracePeriodSeconds: 10
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503, so the pod leaves the Service endpoints before it stops
|
||||
# listening. 0 stops at once.
|
||||
preStopSeconds: 0
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
# add additional https port
|
||||
@@ -1318,7 +1331,8 @@ s3:
|
||||
|
||||
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
|
||||
# the Service endpoints before it receives SIGTERM; keep it shorter than
|
||||
# terminationGracePeriodSeconds.
|
||||
# terminationGracePeriodSeconds. s3.preStopSeconds does the same from
|
||||
# inside the gateway and also fails its health checks.
|
||||
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
# Example:
|
||||
# lifecycle:
|
||||
@@ -2011,6 +2025,7 @@ allInOne:
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables)
|
||||
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
|
||||
@@ -45,6 +45,8 @@ Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
- No quorum among filers: 2 already survive one failure; the third keeps a spare while one is down for upgrades
|
||||
- Give clients every filer address (e.g. `-filer=f1:8888,f2:8888,f3:8888`) or a load balancer that forwards both the HTTP and gRPC ports (8888 and 18888)
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
@@ -77,7 +79,7 @@ Keeps housekeeping off the hot path.
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Filer | 3 (2 minimum) | Metadata gateway, no quorum |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
|
||||
+1055
-1084
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 55 KiB After Width: | Height: | Size: 54 KiB |
@@ -2302,6 +2302,113 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Writable Volumes per Disk",
|
||||
"description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.",
|
||||
"type": "timeseries",
|
||||
"id": 262,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{collection}} disks with 2+"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "{{collection}} max on one disk"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -6629,6 +6736,423 @@
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribe Delivery Lag",
|
||||
"description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope",
|
||||
"type": "timeseries",
|
||||
"id": 263,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Watermark Hold Duration",
|
||||
"description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)",
|
||||
"type": "timeseries",
|
||||
"id": 264,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Delivery Claim / Peer Low-Watermark Age",
|
||||
"description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled",
|
||||
"type": "timeseries",
|
||||
"id": 265,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "claim {{instance}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "peer low {{kind}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribers Resumed from Disk",
|
||||
"description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log",
|
||||
"type": "timeseries",
|
||||
"id": 266,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{scope}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -6642,7 +7166,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
"y": 87
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -6738,7 +7262,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 71
|
||||
"y": 87
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -6834,7 +7358,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
"y": 95
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -6930,7 +7454,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
"y": 95
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -10263,6 +10787,102 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Time Since Last Full Lifecycle Pass",
|
||||
"description": "Time since a pass covered every shard and lifecycle policy without an error; never excluded",
|
||||
"type": "timeseries",
|
||||
"id": 261,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 51
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "time() - max(SeaweedFS_s3_lifecycle_daily_run_last_full_pass_timestamp_seconds{cluster=~\"$cluster\"} > 0)",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "last full pass"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -908,14 +908,9 @@ impl VectorService for VectorServer {
|
||||
0 => DEFAULT_PAGE,
|
||||
n => n.min(MAX_PAGE),
|
||||
};
|
||||
// A purge keeps only the latest version, so lease the one this page reads.
|
||||
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
|
||||
if segments > 1 {
|
||||
// Data and metadata come from a second read of this version, so
|
||||
// lease it as get_vectors does.
|
||||
let _lease = if columns.len() > 1 {
|
||||
self.leases.try_pin(&dataset).await.map_err(internal)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (vectors, next) = list_segment(
|
||||
&dataset,
|
||||
&columns,
|
||||
|
||||
Generated
+2
-2
@@ -3195,9 +3195,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redb"
|
||||
version = "4.2.0"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36"
|
||||
checksum = "096b1dd5ec053c9c2a5a26a93aafeadf73ebcb710ea54dc3151f17a8e20535ef"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -19,9 +19,6 @@ path = "src/main.rs"
|
||||
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
|
||||
default = ["5bytes"]
|
||||
5bytes = []
|
||||
# Unstable redb cursor bulk-load for full_rebuild. Off in production.
|
||||
# Pulls redb's experimental_cursor (and therefore experimental-api-5).
|
||||
redb-experimental-cursor = ["redb/experimental_cursor"]
|
||||
|
||||
[lints.clippy]
|
||||
# Protobuf message literals keep `..Default::default()` on purpose: it is
|
||||
|
||||
@@ -147,6 +147,7 @@ service VolumeServer {
|
||||
message BatchDeleteRequest {
|
||||
repeated string file_ids = 1;
|
||||
bool skip_cookie_check = 2;
|
||||
bool fsync = 3;
|
||||
}
|
||||
|
||||
message BatchDeleteResponse {
|
||||
|
||||
@@ -138,7 +138,8 @@ pub struct Cli {
|
||||
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_upload_limit_mb: u32,
|
||||
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited.
|
||||
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
|
||||
/// the limit are sent to a replica unless readMode is local.
|
||||
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
|
||||
pub concurrent_download_limit_mb: u32,
|
||||
|
||||
|
||||
@@ -416,6 +416,7 @@ async fn run(
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
},
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight,
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
@@ -450,6 +451,7 @@ async fn run(
|
||||
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store.store_dirs = config.folders.clone();
|
||||
for (i, dir) in config.folders.iter().enumerate() {
|
||||
let idx_dir = if config.idx_folder.is_empty() {
|
||||
dir.as_str()
|
||||
|
||||
@@ -106,6 +106,39 @@ pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims,
|
||||
Ok(data.claims)
|
||||
}
|
||||
|
||||
/// Reports a signed, fid-bound write token whose only failure is expiry. It
|
||||
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
|
||||
pub fn is_expired_volume_jwt(signing_key: &SigningKey, token: &str, expected_fid: &str) -> bool {
|
||||
match decode_jwt(signing_key, token) {
|
||||
Err(JwtError::Jwt(e))
|
||||
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
|
||||
_ => return false,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ExpiredClaims {
|
||||
fid: Option<String>,
|
||||
iat: Option<u64>,
|
||||
}
|
||||
let mut validation = Validation::new(Algorithm::HS256);
|
||||
validation.required_spec_claims.clear();
|
||||
validation.validate_exp = false;
|
||||
validation.validate_nbf = true;
|
||||
validation.leeway = 0;
|
||||
let Ok(data) = decode::<ExpiredClaims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(&signing_key.0),
|
||||
&validation,
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Guard
|
||||
// ============================================================================
|
||||
@@ -410,6 +443,53 @@ mod tests {
|
||||
assert!(decode_jwt(&key2, &token).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_expired_volume_jwt_classifies_expiry_only() {
|
||||
let key = SigningKey::from_string("write-key");
|
||||
let fid = "3,01637037d6";
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let sign = |claims: serde_json::Value, key: &SigningKey| {
|
||||
encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&claims,
|
||||
&EncodingKey::from_secret(&key.0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
|
||||
assert!(is_expired_volume_jwt(&key, &expired, fid));
|
||||
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7"));
|
||||
assert!(!is_expired_volume_jwt(
|
||||
&SigningKey::from_string("other-key"),
|
||||
&expired,
|
||||
fid
|
||||
));
|
||||
assert!(!is_expired_volume_jwt(&SigningKey(vec![]), &expired, fid));
|
||||
let immature = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &immature, fid));
|
||||
let future_issued = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &future_issued, fid));
|
||||
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
|
||||
assert!(!is_expired_volume_jwt(&key, &live, fid));
|
||||
let unsigned = format!(
|
||||
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
|
||||
base64::Engine::encode(
|
||||
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
|
||||
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
|
||||
)
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &unsigned, fid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_empty_whitelist() {
|
||||
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -483,6 +483,18 @@ fn extract_file_id(path: &str) -> String {
|
||||
format!("{},{}", parsed.vid, fid)
|
||||
}
|
||||
|
||||
fn is_multipart_form_data(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.split(';').next())
|
||||
.is_some_and(|media_type| {
|
||||
media_type
|
||||
.trim()
|
||||
.eq_ignore_ascii_case("multipart/form-data")
|
||||
})
|
||||
}
|
||||
|
||||
fn streaming_chunk_size(read_buffer_size_bytes: usize, data_size: usize) -> usize {
|
||||
std::cmp::min(
|
||||
read_buffer_size_bytes.max(DEFAULT_STREAMING_CHUNK_SIZE),
|
||||
@@ -671,6 +683,7 @@ async fn batch_delete_file_ids(
|
||||
.batch_delete(volume_server_pb::BatchDeleteRequest {
|
||||
file_ids: batch.clone(),
|
||||
skip_cookie_check: true,
|
||||
fsync: false,
|
||||
})
|
||||
.await
|
||||
.map_err(|e| format!("batch delete {}: {}", grpc_addr, e))?
|
||||
@@ -913,7 +926,6 @@ async fn proxy_or_redirect_to_target(
|
||||
state: &VolumeServerState,
|
||||
info: ProxyRequestInfo,
|
||||
vid: VolumeId,
|
||||
allow_local_redirect: bool,
|
||||
) -> Response {
|
||||
// Look up volume locations from master
|
||||
let locations = match lookup_volume(
|
||||
@@ -961,10 +973,7 @@ async fn proxy_or_redirect_to_target(
|
||||
match state.read_mode {
|
||||
ReadMode::Proxy => proxy_request(state, &info, target).await,
|
||||
ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme),
|
||||
ReadMode::Local if allow_local_redirect => {
|
||||
redirect_request(&info, target, &state.outgoing_http_scheme)
|
||||
}
|
||||
ReadMode::Local => unreachable!(),
|
||||
ReadMode::Local => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1431,7 +1440,7 @@ async fn proxy_missing_volume(
|
||||
None => return StatusCode::NOT_FOUND.into_response(),
|
||||
};
|
||||
|
||||
proxy_or_redirect_to_target(state, info, vid, false).await
|
||||
proxy_or_redirect_to_target(state, info, vid).await
|
||||
}
|
||||
|
||||
/// Go's `proxied` query param check (loop prevention).
|
||||
@@ -1464,8 +1473,10 @@ async fn check_download_limit(
|
||||
|
||||
// Go tries proxy to replica once before the wait loop
|
||||
// (checkDownloadLimit); it does not retry on each wakeup.
|
||||
let should_try_replica =
|
||||
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
|
||||
let should_try_replica = state.read_mode != ReadMode::Local
|
||||
&& !is_proxied_query(&query_string)
|
||||
&& !state.current_master().await.is_empty()
|
||||
&& {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
@@ -1475,25 +1486,24 @@ async fn check_download_limit(
|
||||
&& let Some(info) =
|
||||
build_proxy_request_info(path, request_headers, &query_string, method)
|
||||
{
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
|
||||
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await);
|
||||
}
|
||||
|
||||
// Blocking wait loop (Go's waitForDownloadSlot)
|
||||
loop {
|
||||
if tokio::time::timeout_at(deadline, state.download_notify.notified())
|
||||
.await
|
||||
.is_err()
|
||||
let notified = state.download_notify.notified();
|
||||
if state.inflight_download_bytes.load(Ordering::Relaxed)
|
||||
<= state.concurrent_download_limit
|
||||
{
|
||||
break;
|
||||
}
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
return ControlFlow::Break(json_error_with_query(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"download limit exceeded",
|
||||
uri.query(),
|
||||
));
|
||||
}
|
||||
let current = state.inflight_download_bytes.load(Ordering::Relaxed);
|
||||
if current <= state.concurrent_download_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
ControlFlow::Continue(true)
|
||||
@@ -2780,14 +2790,28 @@ pub async fn post_handler(
|
||||
// JWT check for writes
|
||||
let file_id = extract_file_id(&path);
|
||||
let token = extract_jwt(&headers, request.uri());
|
||||
if state
|
||||
.guard
|
||||
.read()
|
||||
.unwrap()
|
||||
.check_jwt_for_file(token.as_deref(), &file_id, true)
|
||||
.is_err()
|
||||
{
|
||||
return json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
|
||||
let expired_jwt = {
|
||||
let guard = state.guard.read().unwrap();
|
||||
match guard.check_jwt_for_file(token.as_deref(), &file_id, true) {
|
||||
Ok(()) => None,
|
||||
Err(_) => Some(
|
||||
is_multipart_form_data(&headers)
|
||||
&& token.as_deref().is_some_and(|token| {
|
||||
crate::security::is_expired_volume_jwt(&guard.signing_key, token, &file_id)
|
||||
}),
|
||||
),
|
||||
}
|
||||
};
|
||||
if let Some(expired_jwt) = expired_jwt {
|
||||
let mut response =
|
||||
json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
|
||||
if expired_jwt {
|
||||
response.headers_mut().insert(
|
||||
"x-seaweed-write-outcome",
|
||||
header::HeaderValue::from_static("expired-jwt-before-append"),
|
||||
);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
// Upload throttling: check inflight bytes against limit
|
||||
@@ -3657,11 +3681,16 @@ pub async fn delete_handler(
|
||||
|
||||
// Read needle first to validate cookie (matching Go behavior)
|
||||
let original_cookie = cookie;
|
||||
let is_replicate = serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.find(|(name, _)| name == "type")
|
||||
.is_some_and(|(_, value)| value == "replicate");
|
||||
let del_fields =
|
||||
serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query).unwrap_or_default();
|
||||
let del_field = |key: &str| {
|
||||
del_fields
|
||||
.iter()
|
||||
.find(|(name, _)| name == key)
|
||||
.map(|(_, value)| value.as_str())
|
||||
};
|
||||
let is_replicate = del_field("type") == Some("replicate");
|
||||
let fsync = del_field("fsync") == Some("true");
|
||||
{
|
||||
let vol = {
|
||||
let store = state.store.read().unwrap();
|
||||
@@ -3763,7 +3792,16 @@ pub async fn delete_handler(
|
||||
Ok(vol) if vol.is_no_write_or_delete() => {
|
||||
Err(crate::storage::volume::VolumeError::ReadOnly(vid))
|
||||
}
|
||||
Ok(vol) => vol.delete_needle(&mut n),
|
||||
Ok(vol) if fsync => {
|
||||
tokio::task::spawn_blocking(move || vol.delete_needle(&mut n, true))
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
Err(crate::storage::volume::VolumeError::Io(
|
||||
std::io::Error::other(e),
|
||||
))
|
||||
})
|
||||
}
|
||||
Ok(vol) => vol.delete_needle(&mut n, false),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5468,6 +5506,83 @@ mod tests {
|
||||
assert_eq!(extract_jwt(&headers, &uri), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_expired_write_jwt_is_marked_before_append() {
|
||||
use crate::security::SigningKey;
|
||||
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
|
||||
use tower::ServiceExt;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let state = volume_test_state(&tmp);
|
||||
let key = SigningKey(b"write-secret".to_vec());
|
||||
state.guard.write().unwrap().signing_key = key.clone();
|
||||
let volume = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
let before = volume.dat_file_size().unwrap();
|
||||
let fid = format!("1,{:x}{TEST_COOKIE:08x}", 0x6e7a_0c09u64);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let sign = |key: &[u8]| {
|
||||
encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&serde_json::json!({"fid": fid, "exp": now - 10}),
|
||||
&EncodingKey::from_secret(key),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let boundary = "expiredjwtboundary";
|
||||
let body = format!(
|
||||
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a\"\r\n\r\nexpired-bytes\r\n--{boundary}--\r\n"
|
||||
);
|
||||
for (token, content_type, marked) in [
|
||||
(
|
||||
sign(&key.0),
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
true,
|
||||
),
|
||||
(
|
||||
sign(b"other-secret"),
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
false,
|
||||
),
|
||||
(sign(&key.0), "application/octet-stream".to_string(), false),
|
||||
] {
|
||||
let response = super::super::volume_server::build_admin_router(state.clone())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(format!("/{fid}"))
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.header(header::CONTENT_TYPE, &content_type)
|
||||
.body(Body::from(body.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"{content_type}"
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-seaweed-write-outcome")
|
||||
.map(|value| value.to_str().unwrap()),
|
||||
marked.then_some("expired-jwt-before-append"),
|
||||
"{content_type}"
|
||||
);
|
||||
}
|
||||
assert_eq!(volume.dat_file_size().unwrap(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_handle_range_single() {
|
||||
let data = b"hello world";
|
||||
@@ -6281,6 +6396,7 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
@@ -6362,6 +6478,88 @@ mod tests {
|
||||
test_state_with_store(store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_local_read_mode_waits_instead_of_proxying() {
|
||||
use crate::storage::super_block::ReplicaPlacement;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mut state = volume_test_state(&tmp);
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Local;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_millis(50);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
|
||||
.unwrap();
|
||||
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
let flow = check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await;
|
||||
match flow {
|
||||
ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS),
|
||||
ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_limit_wait_sees_release_before_waiting() {
|
||||
let mut state = streaming_test_state();
|
||||
{
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
s.read_mode = ReadMode::Proxy;
|
||||
s.concurrent_download_limit = 1;
|
||||
s.inflight_download_data_timeout = std::time::Duration::from_secs(5);
|
||||
}
|
||||
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
state.inflight_download_bytes.store(2, Ordering::Relaxed);
|
||||
|
||||
let master_guard = state.current_master_url.write().await;
|
||||
let waiter = tokio::spawn({
|
||||
let state = state.clone();
|
||||
async move {
|
||||
let path = "/1,0123456789abcdef";
|
||||
let uri: axum::http::Uri = path.parse().unwrap();
|
||||
check_download_limit(
|
||||
&state,
|
||||
&uri,
|
||||
&HeaderMap::new(),
|
||||
path,
|
||||
VolumeId(1),
|
||||
&Method::GET,
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
state.inflight_download_bytes.store(0, Ordering::Relaxed);
|
||||
state.download_notify.notify_waiters();
|
||||
drop(master_guard);
|
||||
|
||||
let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
|
||||
.await
|
||||
.expect("waiter missed the released download slot")
|
||||
.unwrap();
|
||||
assert!(matches!(flow, ControlFlow::Continue(true)));
|
||||
}
|
||||
|
||||
/// Write a needle to volume 1 and return its URL path.
|
||||
fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
|
||||
put_test_needle_with(state, id, data, |_| {})
|
||||
@@ -8577,7 +8775,7 @@ mod tests {
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.delete_volume_needle(VolumeId(1), &mut filler)
|
||||
.delete_volume_needle(VolumeId(1), &mut filler, false)
|
||||
.unwrap();
|
||||
let data_offset = || {
|
||||
let mut plan = crate::storage::store::Store::needle_read_plan(
|
||||
|
||||
@@ -1698,6 +1698,7 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
|
||||
@@ -2384,23 +2384,45 @@ pub(crate) async fn recover_missing_ec_indexes(
|
||||
"cannot look up peers to recover missing .ecx: {}",
|
||||
e
|
||||
);
|
||||
continue;
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
if peers.is_empty() {
|
||||
tracing::warn!(
|
||||
volume_id = m.vid.0,
|
||||
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
|
||||
);
|
||||
continue;
|
||||
// Under the volume's lifecycle lock, so an in-flight VolumeEcShardsCopy
|
||||
// of it, which may unlink the shards it renamed in, never has them
|
||||
// mounted from under it.
|
||||
let _lifecycle = state.ec_lifecycle_lock(m.vid).lock_owned().await;
|
||||
let current = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.collect_ec_volumes_missing_index_only(Some(m.vid))
|
||||
.into_iter()
|
||||
.find(|x| x.vid == m.vid && x.collection == m.collection);
|
||||
// A copy holding the lock may have installed the index meanwhile: skip
|
||||
// the fetch, but its shards still need the mount.
|
||||
if let Some(current) = ¤t {
|
||||
if peers.is_empty() {
|
||||
tracing::warn!(
|
||||
volume_id = m.vid.0,
|
||||
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !fetch_ec_index_from_peers(state, current, &peers).await {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if fetch_ec_index_from_peers(state, m, &peers).await {
|
||||
state
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.mount_recovered_ec_shards_only(Some(m.vid));
|
||||
if current.is_some() {
|
||||
recovered += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if recovered > 0 {
|
||||
state.store.write().unwrap().mount_recovered_ec_shards();
|
||||
tracing::info!(
|
||||
"recovered missing EC index for {} volume(s) from peers and mounted their shards",
|
||||
recovered
|
||||
|
||||
@@ -125,6 +125,11 @@ pub struct VolumeServerState {
|
||||
pub cli_white_list: Vec<String>,
|
||||
/// Path to state.pb file for persisting VolumeServerState across restarts.
|
||||
pub state_file_path: String,
|
||||
/// Per-vid EC lifecycle lock, serializing EC ops that install or remove a
|
||||
/// vid's files. A tokio Mutex so the guard survives the copy's `.await` IO.
|
||||
pub ec_lifecycle_locks: std::sync::Mutex<
|
||||
std::collections::HashMap<crate::storage::types::VolumeId, Arc<tokio::sync::Mutex<()>>>,
|
||||
>,
|
||||
/// Volumes with an EC decode in flight. A dropped request leaves the
|
||||
/// blocking job running; this keeps a retry from racing it on the
|
||||
/// same volume files.
|
||||
@@ -151,6 +156,20 @@ pub struct VolumeServerState {
|
||||
}
|
||||
|
||||
impl VolumeServerState {
|
||||
/// Return the per-vid EC lifecycle lock, creating it on first use. Entries
|
||||
/// are never reaped.
|
||||
pub fn ec_lifecycle_lock(
|
||||
&self,
|
||||
vid: crate::storage::types::VolumeId,
|
||||
) -> Arc<tokio::sync::Mutex<()>> {
|
||||
self.ec_lifecycle_locks
|
||||
.lock()
|
||||
.unwrap()
|
||||
.entry(vid)
|
||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// The master lookups should go to: the leader the heartbeat confirmed,
|
||||
/// or empty while none is known, in its announced `host:port[.grpcPort]`
|
||||
/// form. Mirrors Go's `vs.getCurrentMaster()`.
|
||||
|
||||
@@ -467,6 +467,7 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
|
||||
@@ -58,6 +58,8 @@ pub struct DiskLocation {
|
||||
/// skips a claimed vid outright and claims any other through its load,
|
||||
/// so a VolumeCopy's .note sweep cannot unlink files it still writes.
|
||||
pub(crate) load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
|
||||
/// The data directory of every disk on this server.
|
||||
pub(crate) store_dirs: Vec<String>,
|
||||
}
|
||||
|
||||
/// Keeps `vid` in the copy-claim set through one scan load so a VolumeCopy
|
||||
@@ -123,6 +125,7 @@ impl DiskLocation {
|
||||
disk_free_bytes: AtomicU64::new(0),
|
||||
min_free_space,
|
||||
load_guard: None,
|
||||
store_dirs: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -486,8 +489,20 @@ impl DiskLocation {
|
||||
|
||||
// Custom ratio comes from the volume's own .vif; the server holds no
|
||||
// cluster EC config in memory.
|
||||
let data_shards =
|
||||
ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid);
|
||||
let data_shards = ec_data_shards_from_vif(
|
||||
&self.directory,
|
||||
&self.idx_directory,
|
||||
&self.store_dirs,
|
||||
collection,
|
||||
vid,
|
||||
);
|
||||
if data_shards == 0 {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"sibling .vif files disagree on the encode; keeping EC shards"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
// On-disk .dat: an empty <= superblock one is a stub; a transient stat
|
||||
// error keeps the shards rather than deleting.
|
||||
@@ -814,6 +829,24 @@ impl DiskLocation {
|
||||
self.volumes.len()
|
||||
}
|
||||
|
||||
/// Number of local volumes of `collection` that still take writes: not
|
||||
/// read-only and, when `volume_size_limit` is known (non zero), with a .dat
|
||||
/// below it. The .dat size is what the master compares with the limit to
|
||||
/// stop writes. Mirrors Go's WritableCollectionVolumesLen.
|
||||
pub fn writable_collection_volumes_len(
|
||||
&self,
|
||||
collection: &str,
|
||||
volume_size_limit: u64,
|
||||
) -> usize {
|
||||
self.volumes
|
||||
.values()
|
||||
.filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only())
|
||||
.filter(|v| {
|
||||
volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit
|
||||
})
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Get all volume IDs, sorted.
|
||||
pub fn volume_ids(&self) -> Vec<VolumeId> {
|
||||
let mut ids: Vec<VolumeId> = self.volumes.keys().copied().collect();
|
||||
@@ -1020,7 +1053,9 @@ impl DiskLocation {
|
||||
// propagate the error to the caller.
|
||||
let created = !self.ec_volumes.contains_key(&vid);
|
||||
if created {
|
||||
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid).map_err(VolumeError::Io)?;
|
||||
let ec_vol =
|
||||
EcVolume::new_with_vif_dirs(&dir, idx_dir, &self.store_dirs, collection, vid)
|
||||
.map_err(VolumeError::Io)?;
|
||||
self.ec_volumes.insert(vid, ec_vol);
|
||||
}
|
||||
let ec_vol = self.ec_volumes.get_mut(&vid).expect("just inserted above");
|
||||
@@ -1441,6 +1476,7 @@ fn rm_if_present(path: String) -> io::Result<()> {
|
||||
fn ec_data_shards_from_vif(
|
||||
directory: &str,
|
||||
idx_directory: &str,
|
||||
store_dirs: &[String],
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> usize {
|
||||
@@ -1459,6 +1495,23 @@ fn ec_data_shards_from_vif(
|
||||
break;
|
||||
}
|
||||
}
|
||||
match crate::storage::erasure_coding::ec_volume::sibling_ec_config_vif(
|
||||
directory,
|
||||
idx_directory,
|
||||
store_dirs,
|
||||
collection,
|
||||
vid,
|
||||
) {
|
||||
Err(()) => return 0,
|
||||
Ok(Some(vi)) => {
|
||||
if let Some(ds) = vi.ec_shard_config.map(|c| c.data_shards as usize)
|
||||
&& ds > 0
|
||||
{
|
||||
return ds;
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
}
|
||||
DATA_SHARDS_COUNT
|
||||
}
|
||||
|
||||
@@ -1542,7 +1595,7 @@ pub(crate) fn vif_references_remote_file(vif_path: &str) -> bool {
|
||||
/// True when a `.vif` records an EC shard config, i.e. the volume was
|
||||
/// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat`
|
||||
/// alongside it is a leftover stub safe to remove.
|
||||
fn vif_is_ec_volume(vif_path: &str) -> bool {
|
||||
pub(crate) fn vif_is_ec_volume(vif_path: &str) -> bool {
|
||||
fs::read_to_string(vif_path)
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
|
||||
|
||||
@@ -272,6 +272,96 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume
|
||||
data_vif
|
||||
}
|
||||
|
||||
/// [`locate_vif_path`], except that a data-dir `.vif` recording no EC config
|
||||
/// yields to the idx dir's when that one records it: a source volume's own
|
||||
/// `.vif` stays on a disk that later receives shards of the same volume id,
|
||||
/// while the encode's `.vif` lands with the index on a sibling.
|
||||
fn resolve_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
|
||||
let path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let data_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id),
|
||||
);
|
||||
if dir_idx == dir || path != data_vif {
|
||||
return path;
|
||||
}
|
||||
let idx_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
|
||||
);
|
||||
prefer_ec_config_vif(path, idx_vif)
|
||||
}
|
||||
|
||||
fn read_vif_file(path: &str) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(content) if content.trim().is_empty() => Ok(None),
|
||||
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|_| ()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(_) => Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `alt` when `local` records no EC config but `alt` does.
|
||||
fn prefer_ec_config_vif(local: String, alt: String) -> String {
|
||||
match read_vif_file(&local) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => return local,
|
||||
Err(()) => return local,
|
||||
_ => {}
|
||||
}
|
||||
match read_vif_file(&alt) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => alt,
|
||||
_ => local,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `.vif` a disk mounting against a shared `-dir.idx` takes from a sibling
|
||||
/// data dir in `vif_dirs`: its own records no EC config, the index is not its
|
||||
/// own, and every sibling `.vif` recording an EC config names the same run.
|
||||
/// `Err` when a sibling `.vif` cannot be read or the siblings name different runs.
|
||||
pub(crate) fn sibling_ec_config_vif(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
vif_dirs: &[String],
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let data_vif = format!("{}.vif", data_base);
|
||||
if dir_idx == dir
|
||||
|| vif_dirs.iter().any(|d| d == dir_idx)
|
||||
|| matches!(ecx_file_size(&format!("{}.ecx", data_base)), Some(n) if n > 0)
|
||||
|| resolve_vif_path(dir, dir_idx, collection, volume_id) != data_vif
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
match read_vif_file(&data_vif) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => return Ok(None),
|
||||
Err(()) => return Ok(None),
|
||||
_ => {}
|
||||
}
|
||||
let mut found: Option<crate::storage::volume::VifVolumeInfo> = None;
|
||||
for other in vif_dirs.iter().filter(|d| d.as_str() != dir) {
|
||||
let path = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(other, collection, volume_id)
|
||||
);
|
||||
let info = match read_vif_file(&path) {
|
||||
Ok(Some(info)) if info.ec_shard_config.is_some() => info,
|
||||
Ok(_) => continue,
|
||||
Err(()) => return Err(()),
|
||||
};
|
||||
let run = |v: &crate::storage::volume::VifVolumeInfo| {
|
||||
v.ec_shard_config.as_ref().map(|c| c.encode_ts_ns)
|
||||
};
|
||||
match &found {
|
||||
None => found = Some(info),
|
||||
Some(f) if run(f) != run(&info) => return Err(()),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
/// Load the volume's `.vif` (data dir first, then idx dir, then the active
|
||||
/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
|
||||
/// everywhere is legal — legacy volumes predate the sidecar — and returns
|
||||
@@ -286,7 +376,7 @@ pub fn load_vif_info(
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let vif_path = resolve_vif_path(dir, dir_idx, collection, volume_id);
|
||||
match std::fs::read_to_string(&vif_path) {
|
||||
Ok(content) if content.trim().is_empty() => Ok(None),
|
||||
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
|
||||
@@ -336,15 +426,16 @@ pub fn load_vif_info_across_dirs(
|
||||
// pointed callers at the wrong disk whenever the vif lived with the
|
||||
// index — and a caller resolving the rest of the volume's metadata
|
||||
// against that answer would look in a directory holding none of it.
|
||||
let data_vif = format!(
|
||||
let idx_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id)
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id)
|
||||
);
|
||||
let found_in = if std::path::Path::new(&data_vif).exists() {
|
||||
dir.to_string()
|
||||
} else {
|
||||
dir_idx.to_string()
|
||||
};
|
||||
let found_in =
|
||||
if dir_idx != dir && resolve_vif_path(dir, dir_idx, collection, volume_id) == idx_vif {
|
||||
dir_idx.to_string()
|
||||
} else {
|
||||
dir.to_string()
|
||||
};
|
||||
return Ok(Some((vif, found_in)));
|
||||
}
|
||||
for other in other_dirs {
|
||||
@@ -569,7 +660,25 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL)
|
||||
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL, &[])
|
||||
}
|
||||
|
||||
/// [`Self::new`] for a disk whose server's data directories are `vif_dirs`.
|
||||
pub fn new_with_vif_dirs(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
vif_dirs: &[String],
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
Self::open_with(
|
||||
dir,
|
||||
dir_idx,
|
||||
collection,
|
||||
volume_id,
|
||||
EcjFsOps::REAL,
|
||||
vif_dirs,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`Self::new`] with the journal-compaction filesystem steps supplied, so
|
||||
@@ -580,10 +689,15 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
ecj_ops: EcjFsOps,
|
||||
vif_dirs: &[String],
|
||||
) -> io::Result<Self> {
|
||||
// One load of the volume's `.vif`, used for both the shard config and
|
||||
// the version / dat-size fields below.
|
||||
let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
|
||||
let vif = sibling_ec_config_vif(dir, dir_idx, vif_dirs, collection, volume_id)
|
||||
.ok()
|
||||
.flatten()
|
||||
.or(vif);
|
||||
// Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
|
||||
// the INDEX, and it is the only layout record a volume whose vif is
|
||||
// absent or config-free still has.
|
||||
@@ -3580,7 +3694,7 @@ mod tests {
|
||||
);
|
||||
let before = std::fs::read(&ecj_path).unwrap();
|
||||
(
|
||||
EcVolume::open_with(dir, dir, "", vid, ops),
|
||||
EcVolume::open_with(dir, dir, "", vid, ops, &[]),
|
||||
ecj_path,
|
||||
before,
|
||||
)
|
||||
@@ -3801,7 +3915,7 @@ mod tests {
|
||||
read_at: rewrite_after_last_read,
|
||||
..EcjFsOps::REAL
|
||||
};
|
||||
let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap();
|
||||
let vol = EcVolume::open_with(dir, dir, "", vid, ops, &[]).unwrap();
|
||||
assert!(
|
||||
IN_PLACE_REWRITE.with(|r| r.borrow().is_none()),
|
||||
"the rewrite never ran"
|
||||
@@ -4108,6 +4222,143 @@ mod tests {
|
||||
assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
|
||||
}
|
||||
|
||||
/// Shards placed on the disk that still holds the source volume's `.vif`
|
||||
/// mount against an index on a sibling disk; the encode identity and
|
||||
/// layout come from the sibling's `.vif`, not the source's config-free one.
|
||||
#[test]
|
||||
fn test_ec_volume_prefers_idx_vif_over_config_free_data_vif() {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", data_base),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
dat_file_size: 4096,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 12,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_123_456_789,
|
||||
block_size: 3 * 1024 * 1024,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let idx_base = crate::storage::volume::volume_file_name(dir_idx, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", idx_base),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, 1_700_000_000_123_456_789);
|
||||
assert_eq!((vol.data_shards, vol.parity_shards), (12, 4));
|
||||
assert_eq!(vol.block_size, 3 * 1024 * 1024);
|
||||
assert_eq!(vol.dat_file_size, 4096);
|
||||
let (_, found_in) = load_vif_info_across_dirs(dir, dir_idx, &[], "pics", VolumeId(1))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(found_in, dir_idx);
|
||||
}
|
||||
|
||||
/// A data-dir `.vif` that records an EC config stays authoritative.
|
||||
#[test]
|
||||
fn test_ec_volume_keeps_data_vif_with_ec_config() {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
|
||||
for (d, ts) in [(dir, 11), (dir_idx, 22)] {
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: ts,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let base = crate::storage::volume::volume_file_name(d, "pics", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, 11);
|
||||
}
|
||||
|
||||
/// With a shared `-dir.idx` the encode's `.vif` sits in the data dir of the
|
||||
/// disk that received the index, so a disk holding a source volume's
|
||||
/// config-free `.vif` takes it from that sibling, but only when the
|
||||
/// siblings agree on one run.
|
||||
#[test]
|
||||
fn test_ec_volume_takes_sibling_vif_with_shared_index() {
|
||||
const RUN_A: i64 = 1_700_000_000_000_000_011;
|
||||
const RUN_B: i64 = 1_700_000_000_000_000_022;
|
||||
let cases: [(&str, &[i64], bool, i64); 4] = [
|
||||
("one sibling", &[RUN_A], false, RUN_A),
|
||||
("siblings agree", &[RUN_A, RUN_A], false, RUN_A),
|
||||
("siblings disagree", &[RUN_A, RUN_B], false, 0),
|
||||
("local index", &[RUN_A], true, 0),
|
||||
];
|
||||
for (name, sibling_ts, local_index, want) in cases {
|
||||
let data_tmp = TempDir::new().unwrap();
|
||||
let idx_tmp = TempDir::new().unwrap();
|
||||
let dir = data_tmp.path().to_str().unwrap();
|
||||
let dir_idx = idx_tmp.path().to_str().unwrap();
|
||||
let entry = [(NeedleId(1), Offset::from_actual_offset(8), Size(10))];
|
||||
write_ecx_file(dir_idx, "c", VolumeId(1), &entry);
|
||||
if local_index {
|
||||
write_ecx_file(dir, "c", VolumeId(1), &entry);
|
||||
}
|
||||
let data_base = crate::storage::volume::volume_file_name(dir, "c", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", data_base),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let mut dirs = vec![dir.to_string()];
|
||||
let mut siblings = Vec::new();
|
||||
for &ts in sibling_ts {
|
||||
let sibling = TempDir::new().unwrap();
|
||||
let sibling_dir = sibling.path().to_str().unwrap().to_string();
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
version: 3,
|
||||
dat_file_size: 4096,
|
||||
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: ts,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let base = crate::storage::volume::volume_file_name(&sibling_dir, "c", VolumeId(1));
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
dirs.push(sibling_dir);
|
||||
siblings.push(sibling);
|
||||
}
|
||||
|
||||
let vol = EcVolume::new_with_vif_dirs(dir, dir_idx, &dirs, "c", VolumeId(1)).unwrap();
|
||||
assert_eq!(vol.encode_ts_ns, want, "{}", name);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ec_volume_wide_ratio_vif_config() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -13,7 +13,6 @@ use std::io::{self, Read, Seek, Write};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
||||
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
use std::ops::Bound;
|
||||
|
||||
mod compact_map;
|
||||
@@ -485,6 +484,38 @@ const NEEDLE_TABLE: TableDefinition<u64, &[u8]> = TableDefinition::new("needles"
|
||||
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
|
||||
const META_IDX_SIZE: &str = "idx_size";
|
||||
|
||||
/// redb 4.6 starts a new file at 72 KiB: 64 KiB of pages plus the header
|
||||
/// and the region tracker.
|
||||
const REDB_SMALL_FILE_BYTES: u64 = 72 * 1024;
|
||||
/// redb 4.2 started at 1 MiB of pages plus that same 8 KiB.
|
||||
const REDB_OLD_FLOOR_BYTES: u64 = 1024 * 1024 + 8 * 1024;
|
||||
/// Meta value `1` means a compact of a file in the old-floor band already finished.
|
||||
const META_FLOOR_COMPACT: &str = "floor_compact";
|
||||
|
||||
/// True when a reused `.rdb` is still inside the old 4.2 starting size.
|
||||
/// A file one byte past it has grown.
|
||||
fn should_compact_old_floor(len: u64) -> bool {
|
||||
len > REDB_SMALL_FILE_BYTES && len <= REDB_OLD_FLOOR_BYTES
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CompactFollowUp {
|
||||
/// Drop the `Database` and open the same path once. Do not compact again.
|
||||
ReopenSkipCompact,
|
||||
/// The handle can still begin a write. Do not record `floor_compact`.
|
||||
KeepHandle,
|
||||
}
|
||||
|
||||
fn compact_follow_up(err: &redb::CompactionError) -> CompactFollowUp {
|
||||
match err {
|
||||
redb::CompactionError::Storage(_) => CompactFollowUp::ReopenSkipCompact,
|
||||
redb::CompactionError::TransactionInProgress
|
||||
| redb::CompactionError::PersistentSavepointExists
|
||||
| redb::CompactionError::EphemeralSavepointExists => CompactFollowUp::KeepHandle,
|
||||
_ => CompactFollowUp::KeepHandle,
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes between two durable redb checkpoints. Every non-durable commit
|
||||
/// leaves an entry in redb's transaction tracker and pins the pages that
|
||||
/// were on disk at the last durable commit; only a durable commit clears
|
||||
@@ -648,9 +679,6 @@ impl RedbNeedleMap {
|
||||
let meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
||||
// ReadableTable::get guard borrows `meta`; edition 2024 drops the tail
|
||||
// expression's temporaries before `meta`, so no extra binding is needed.
|
||||
match meta.get(META_IDX_SIZE) {
|
||||
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
@@ -658,6 +686,118 @@ impl RedbNeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
fn rdb_file_len(&self) -> io::Result<u64> {
|
||||
Ok(std::fs::metadata(&self.rdb_path)?.len())
|
||||
}
|
||||
|
||||
fn read_floor_compact(&self) -> io::Result<Option<u64>> {
|
||||
let txn = self
|
||||
.db_or_err()?
|
||||
.begin_read()
|
||||
.map_err(|e| io::Error::other(format!("redb begin_read: {}", e)))?;
|
||||
let meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
match meta.get(META_FLOOR_COMPACT) {
|
||||
Ok(Some(guard)) => Ok(Some(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(io::Error::other(format!("redb get meta: {}", e))),
|
||||
}
|
||||
}
|
||||
|
||||
fn save_floor_compact(&self) -> io::Result<()> {
|
||||
let txn = Self::begin_write_no_fsync(self.db_or_err()?)?;
|
||||
{
|
||||
let mut meta = txn
|
||||
.open_table(META_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
|
||||
meta.insert(META_FLOOR_COMPACT, 1u64)
|
||||
.map_err(|e| io::Error::other(format!("redb insert meta: {}", e)))?;
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit meta: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop the exclusive lock and open the same file. Does not compact.
|
||||
fn reopen_database(&mut self) -> io::Result<()> {
|
||||
self.db.take();
|
||||
let db = Database::builder()
|
||||
.set_cache_size(self.cache_bytes)
|
||||
.open(&self.rdb_path)
|
||||
.map_err(|e| io::Error::other(format!("redb reopen after compact: {}", e)))?;
|
||||
self.db = Some(db);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_floor_compact_if_still_in_band(&self) -> io::Result<()> {
|
||||
let len = match self.rdb_file_len() {
|
||||
Ok(len) => len,
|
||||
Err(e) => {
|
||||
tracing::warn!("redb floor compact flag skipped, metadata failed: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
if should_compact_old_floor(len) {
|
||||
self.save_floor_compact()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact a reused file that is still the 4.2 starting allocation.
|
||||
///
|
||||
/// Called only from `try_reuse_rdb`, after the replay transaction has
|
||||
/// committed. `new` and `full_rebuild` already create a 72 KiB file.
|
||||
///
|
||||
/// An empty durable commit publishes a `Durability::None` tail before
|
||||
/// `compact`. A storage error inside `compact` would otherwise drop that
|
||||
/// tail when the handle is reopened. `checkpoint` is not used: during
|
||||
/// reuse `idx_file` is `None` and `idx_file_offset` is 0.
|
||||
fn shrink_old_floor(&mut self) -> io::Result<()> {
|
||||
let len = match self.rdb_file_len() {
|
||||
Ok(len) => len,
|
||||
Err(e) => {
|
||||
tracing::warn!("redb floor compact skipped, metadata failed: {}", e);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
// The flag is irrelevant outside the band. A meta read error there
|
||||
// must not make `load_from_idx` rebuild a larger index.
|
||||
if !should_compact_old_floor(len) {
|
||||
return Ok(());
|
||||
}
|
||||
if self.read_floor_compact()? == Some(1) {
|
||||
return Ok(());
|
||||
}
|
||||
// Failure returns Err so `load_from_idx` full-rebuilds. Do not reopen
|
||||
// over a tail that this commit did not publish.
|
||||
let mut txn = self
|
||||
.db_or_err()?
|
||||
.begin_write()
|
||||
.map_err(|e| io::Error::other(format!("redb begin_write: {}", e)))?;
|
||||
txn.set_quick_repair(true);
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||
let compact_result = self
|
||||
.db
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("redb database is closed"))?
|
||||
.compact();
|
||||
match compact_result {
|
||||
Ok(_) => self.record_floor_compact_if_still_in_band(),
|
||||
Err(e) => match compact_follow_up(&e) {
|
||||
CompactFollowUp::ReopenSkipCompact => {
|
||||
tracing::warn!("redb floor compact storage error, reopening: {}", e);
|
||||
self.reopen_database()
|
||||
}
|
||||
CompactFollowUp::KeepHandle => {
|
||||
tracing::warn!("redb floor compact skipped: {}", e);
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only read of META `idx_size` through the live handle. See
|
||||
/// [`test_support::live_meta_idx_size`] for why durability tests use
|
||||
/// this instead of copying the open `.rdb`.
|
||||
@@ -666,6 +806,11 @@ impl RedbNeedleMap {
|
||||
self.read_idx_size_meta().unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn live_floor_compact(&self) -> Option<u64> {
|
||||
self.read_floor_compact().unwrap()
|
||||
}
|
||||
|
||||
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
|
||||
///
|
||||
/// Strategy:
|
||||
@@ -774,6 +919,7 @@ impl RedbNeedleMap {
|
||||
nm.save_idx_size_meta(idx_size)?;
|
||||
}
|
||||
|
||||
nm.shrink_old_floor()?;
|
||||
Ok(nm)
|
||||
}
|
||||
|
||||
@@ -796,8 +942,8 @@ impl RedbNeedleMap {
|
||||
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
|
||||
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
|
||||
/// still writable, clear the leftover needles table before inserting.
|
||||
/// Live keys come from a `BTreeMap` so inserts are in needle-id order and
|
||||
/// redb 4.2.0 packs leaves, without a second Vec + sort scratch.
|
||||
/// Live keys come from a `BTreeMap` so `insert_before` walks them in
|
||||
/// needle-id order and packs leaves, without a second Vec + sort scratch.
|
||||
fn full_rebuild<R: Read + Seek>(
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
@@ -849,43 +995,26 @@ impl RedbNeedleMap {
|
||||
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "redb-experimental-cursor"))]
|
||||
{
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
table
|
||||
.insert(key_u64, packed.as_slice())
|
||||
.map_err(|e| io::Error::other(format!("redb insert: {}", e)))?;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
{
|
||||
let mut cursor =
|
||||
table
|
||||
.upper_bound_mut(Bound::<u64>::Unbounded)
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb upper_bound_mut: {}", e),
|
||||
)
|
||||
})?;
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
cursor
|
||||
.insert_before(key_u64, packed.as_slice())
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb insert_before: {}", e),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
cursor.close().map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
||||
let mut cursor = table
|
||||
.upper_bound_mut(Bound::<u64>::Unbounded)
|
||||
.map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb upper_bound_mut: {}", e))
|
||||
})?;
|
||||
for (key, nv) in &entries {
|
||||
let key_u64: u64 = (*key).into();
|
||||
let packed = pack_needle_value(nv);
|
||||
cursor
|
||||
.insert_before(key_u64, packed.as_slice())
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("redb insert_before: {}", e),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
cursor.close().map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
|
||||
})?;
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
|
||||
@@ -1048,9 +1177,6 @@ impl RedbNeedleMap {
|
||||
let table = txn
|
||||
.open_table(NEEDLE_TABLE)
|
||||
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
|
||||
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
|
||||
// ReadableTable::get guard borrows `table`; edition 2024 drops the tail
|
||||
// expression's temporaries before `table`, so no extra binding is needed.
|
||||
match table.get(key_u64) {
|
||||
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
|
||||
Ok(None) => Ok(None),
|
||||
@@ -1181,7 +1307,7 @@ impl RedbNeedleMap {
|
||||
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
|
||||
/// commit error.
|
||||
///
|
||||
/// redb 4.2.0 can make a `Durability::None` commit visible *before*
|
||||
/// redb can make a `Durability::None` commit visible *before*
|
||||
/// returning `Err(CommitError::Storage(..))`. In that state the database
|
||||
/// refuses further write transactions, so the map cannot be used as-is.
|
||||
/// The `.idx` row that triggered the commit is **not** truncated — the
|
||||
@@ -1603,9 +1729,10 @@ pub(crate) mod test_support {
|
||||
/// every `put`/`delete` also commits non-durably, so before the first
|
||||
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
|
||||
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
|
||||
/// A live read is the only portable observation: redb 4.2.0 takes an
|
||||
/// exclusive whole-file lock, which is advisory on Unix but mandatory
|
||||
/// on Windows, so copying the open `.rdb` fails there with OS error 33.
|
||||
/// A live read is the only portable observation: redb's default
|
||||
/// `ExclusiveWriter` takes an exclusive whole-file lock, which is
|
||||
/// advisory on Unix but mandatory on Windows, so copying the open
|
||||
/// `.rdb` fails there with OS error 33.
|
||||
///
|
||||
/// It still pins the property under test: the only *durable* META
|
||||
/// writer is `checkpoint`, so any value other than the load-time one
|
||||
@@ -1763,6 +1890,41 @@ mod tests {
|
||||
NeedleMapKind::Redb.redb_cache_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_new_file_is_at_most_72kib() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db_path = dir.path().join("fresh.rdb");
|
||||
let nm = RedbNeedleMap::new(db_path.to_str().unwrap(), redb_test_cache()).unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(
|
||||
len <= REDB_SMALL_FILE_BYTES,
|
||||
"fresh redb file is {len} bytes, above the 72 KiB start"
|
||||
);
|
||||
drop(nm);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_compact_old_floor_bounds() {
|
||||
let small = REDB_SMALL_FILE_BYTES;
|
||||
let floor = REDB_OLD_FLOOR_BYTES;
|
||||
assert!(!should_compact_old_floor(small));
|
||||
assert!(should_compact_old_floor(small + 1));
|
||||
assert!(should_compact_old_floor(floor));
|
||||
assert!(!should_compact_old_floor(floor + 1));
|
||||
assert!(!should_compact_old_floor(0));
|
||||
}
|
||||
|
||||
fn reload_redb(db_path: &std::path::Path, idx_path: &std::path::Path) -> RedbNeedleMap {
|
||||
let mut idx = std::fs::File::open(idx_path).unwrap();
|
||||
RedbNeedleMap::load_from_idx(
|
||||
db_path.to_str().unwrap(),
|
||||
&mut idx,
|
||||
Version::current(),
|
||||
redb_test_cache(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Open a redb map on an empty .idx with an append writer attached, the
|
||||
/// way `Volume::load_index_redb` opens a writable volume.
|
||||
fn open_writable_redb(
|
||||
@@ -1895,7 +2057,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "redb-experimental-cursor")]
|
||||
#[test]
|
||||
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
|
||||
// Enough keys to split leaves. insert_before vs table.insert only
|
||||
@@ -2474,4 +2635,164 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_fresh_reuse_does_not_record_floor_compact() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(len <= REDB_SMALL_FILE_BYTES, "fresh file is {len} bytes");
|
||||
nm.close();
|
||||
drop(nm);
|
||||
let loaded = reload_redb(&db_path, &idx_path);
|
||||
assert_eq!(loaded.live_floor_compact(), None);
|
||||
assert!(loaded.get(NeedleId(1)).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_in_band_file_is_compacted_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
n += 1;
|
||||
nm.put(
|
||||
NeedleId(n),
|
||||
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||
Size(8),
|
||||
)
|
||||
.unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
if should_compact_old_floor(len) {
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
n < 2_000_000,
|
||||
"file grew to {len} bytes without entering the old-floor band"
|
||||
);
|
||||
}
|
||||
nm.checkpoint(true).unwrap();
|
||||
nm.close();
|
||||
drop(nm);
|
||||
|
||||
let mut loaded = reload_redb(&db_path, &idx_path);
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
let flag = loaded.live_floor_compact();
|
||||
if should_compact_old_floor(len) {
|
||||
assert_eq!(flag, Some(1));
|
||||
} else {
|
||||
assert!(len <= REDB_SMALL_FILE_BYTES, "compacted file is {len} bytes");
|
||||
assert_eq!(flag, None);
|
||||
}
|
||||
let first = loaded.get(NeedleId(1)).unwrap();
|
||||
assert!(first.is_some());
|
||||
let last = loaded.get(NeedleId(n)).unwrap();
|
||||
assert!(last.is_some());
|
||||
// close checkpoints the non-durable floor_compact flag, then drop
|
||||
// releases the exclusive lock before the second open.
|
||||
loaded.close();
|
||||
drop(loaded);
|
||||
|
||||
let again = reload_redb(&db_path, &idx_path);
|
||||
assert_eq!(std::fs::metadata(&db_path).unwrap().len(), len);
|
||||
assert_eq!(again.live_floor_compact(), flag);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_sparse_old_floor_shrinks_on_reuse() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
n += 1;
|
||||
nm.put(
|
||||
NeedleId(n),
|
||||
Offset::from_actual_offset((n as i64).saturating_mul(8)),
|
||||
Size(8),
|
||||
)
|
||||
.unwrap();
|
||||
let len = std::fs::metadata(&db_path).unwrap().len();
|
||||
if should_compact_old_floor(len) {
|
||||
break;
|
||||
}
|
||||
assert!(n < 2_000_000, "file is {len} bytes and never entered the band");
|
||||
}
|
||||
assert!(n > 32, "band was reached with only {n} keys");
|
||||
nm.checkpoint(true).unwrap();
|
||||
nm.close();
|
||||
drop(nm);
|
||||
|
||||
let keep = 32u64;
|
||||
let idx = std::fs::OpenOptions::new().write(true).open(&idx_path).unwrap();
|
||||
idx.set_len(keep * NEEDLE_MAP_ENTRY_SIZE as u64).unwrap();
|
||||
drop(idx);
|
||||
|
||||
let db = redb::Database::builder()
|
||||
.set_cache_size(redb_test_cache())
|
||||
.open(db_path.to_str().unwrap())
|
||||
.unwrap();
|
||||
let txn = db.begin_write().unwrap();
|
||||
{
|
||||
let mut table = txn.open_table(NEEDLE_TABLE).unwrap();
|
||||
for key in (keep + 1)..=n {
|
||||
table.remove(key).unwrap();
|
||||
}
|
||||
}
|
||||
{
|
||||
let mut meta = txn.open_table(META_TABLE).unwrap();
|
||||
meta.insert(META_IDX_SIZE, keep * NEEDLE_MAP_ENTRY_SIZE as u64)
|
||||
.unwrap();
|
||||
}
|
||||
txn.commit().unwrap();
|
||||
drop(db);
|
||||
|
||||
let before = std::fs::metadata(&db_path).unwrap().len();
|
||||
let loaded = reload_redb(&db_path, &idx_path);
|
||||
let after = std::fs::metadata(&db_path).unwrap().len();
|
||||
assert!(after < before, "reuse left the file at {after} bytes, was {before}");
|
||||
assert!(
|
||||
after <= REDB_SMALL_FILE_BYTES,
|
||||
"32 live keys still occupy {after} bytes"
|
||||
);
|
||||
assert!(loaded.get(NeedleId(1)).unwrap().is_some());
|
||||
assert!(loaded.get(NeedleId(keep)).unwrap().is_some());
|
||||
assert!(loaded.get(NeedleId(keep + 1)).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_follow_up_reopens_only_on_storage_error() {
|
||||
use redb::{CompactionError, StorageError};
|
||||
let storage = CompactionError::Storage(StorageError::Io(std::io::Error::other("sync")));
|
||||
assert_eq!(
|
||||
compact_follow_up(&storage),
|
||||
CompactFollowUp::ReopenSkipCompact
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::TransactionInProgress),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::PersistentSavepointExists),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
assert_eq!(
|
||||
compact_follow_up(&CompactionError::EphemeralSavepointExists),
|
||||
CompactFollowUp::KeepHandle
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_redb_reopen_database_reads_committed_key() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut nm, _db_path, _idx_path) = open_writable_redb(dir.path());
|
||||
nm.put(NeedleId(7), Offset::from_actual_offset(64), Size(9))
|
||||
.unwrap();
|
||||
nm.reopen_database().unwrap();
|
||||
let nv = nm.get(NeedleId(7)).unwrap().unwrap();
|
||||
assert_eq!(nv.size, Size(9));
|
||||
nm.put(NeedleId(8), Offset::from_actual_offset(128), Size(10))
|
||||
.unwrap();
|
||||
let nv = nm.get(NeedleId(8)).unwrap().unwrap();
|
||||
assert_eq!(nv.size, Size(10));
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,9 @@ pub struct Store {
|
||||
/// The copy-claim set installed on every location by
|
||||
/// set_volume_load_guard, kept here so a later add_location gets it too.
|
||||
load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
|
||||
/// The data directory of every disk this server loads, set before the
|
||||
/// first `add_location` so a disk's EC mounts can see its siblings.
|
||||
pub store_dirs: Vec<String>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -148,6 +151,7 @@ impl Store {
|
||||
disk_stats_of: super::disk_location::get_disk_stats,
|
||||
load_guard: None,
|
||||
preallocate_supported: PREALLOCATE_SUPPORTED,
|
||||
store_dirs: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,6 +174,7 @@ impl Store {
|
||||
tags,
|
||||
)?;
|
||||
loc.load_guard = self.load_guard.clone();
|
||||
loc.store_dirs = self.store_dirs.clone();
|
||||
loc.load_existing_volumes(self.needle_map_kind)?;
|
||||
|
||||
// Check for duplicate volume IDs across existing locations
|
||||
@@ -287,12 +292,20 @@ impl Store {
|
||||
|
||||
// ---- Volume lifecycle ----
|
||||
|
||||
/// Find the location with fewest volumes (load-balance) of the given disk type.
|
||||
/// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots.
|
||||
fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option<usize> {
|
||||
/// Find the location with the fewest writable volumes of `collection`, so
|
||||
/// they do not share one disk's write bandwidth, then the most free slots
|
||||
/// (load-balance) of the given disk type. Matches Go's FindFreeLocation:
|
||||
/// accounts for EC shards when computing free slots.
|
||||
fn find_free_location(
|
||||
&self,
|
||||
disk_type: &DiskType,
|
||||
collection: &str,
|
||||
preallocate: u64,
|
||||
) -> Option<usize> {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
let mut best: Option<(usize, i64)> = None; // (index, effective_free)
|
||||
let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed);
|
||||
let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free)
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if &loc.disk_type != disk_type {
|
||||
continue;
|
||||
@@ -322,11 +335,14 @@ impl Store {
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if best.is_none() || effective_free > best.unwrap().1 {
|
||||
best = Some((i, effective_free));
|
||||
let writable = loc.writable_collection_volumes_len(collection, volume_size_limit);
|
||||
if best.is_none_or(|(_, min_writable, max_free)| {
|
||||
writable < min_writable || (writable == min_writable && effective_free > max_free)
|
||||
}) {
|
||||
best = Some((i, writable, effective_free));
|
||||
}
|
||||
}
|
||||
best.map(|(i, _)| i)
|
||||
best.map(|(i, _, _)| i)
|
||||
}
|
||||
|
||||
/// Find a free location matching a predicate.
|
||||
@@ -596,7 +612,7 @@ impl Store {
|
||||
return Err(VolumeError::AlreadyExists);
|
||||
}
|
||||
let loc_idx = self
|
||||
.find_free_location(&disk_type, spec.preallocate)
|
||||
.find_free_location(&disk_type, spec.collection, spec.preallocate)
|
||||
.ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::other(format!(
|
||||
"no free location for disk type {:?}",
|
||||
@@ -1037,13 +1053,18 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Delete a needle from a volume.
|
||||
pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
pub fn delete_volume_needle(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
fsync: bool,
|
||||
) -> Result<Size, VolumeError> {
|
||||
// Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
if vol.is_no_write_or_delete() {
|
||||
return Err(VolumeError::ReadOnly(vid));
|
||||
}
|
||||
vol.delete_needle(n)
|
||||
vol.delete_needle(n, fsync)
|
||||
}
|
||||
|
||||
// ---- Collection operations ----
|
||||
@@ -2918,7 +2939,9 @@ mod tests {
|
||||
cookie: Cookie(0xaa),
|
||||
..Needle::default()
|
||||
};
|
||||
let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap();
|
||||
let deleted = store
|
||||
.delete_volume_needle(VolumeId(1), &mut del_n, false)
|
||||
.unwrap();
|
||||
assert!(deleted.0 > 0);
|
||||
}
|
||||
|
||||
@@ -3027,6 +3050,87 @@ mod tests {
|
||||
assert_eq!(store.locations[1].volumes_len(), 1);
|
||||
}
|
||||
|
||||
fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) {
|
||||
let spec = VolumeSpec {
|
||||
collection,
|
||||
..VolumeSpec::default()
|
||||
};
|
||||
store
|
||||
.add_volume(VolumeId(vid), DiskType::HardDrive, &spec)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn location_of(store: &Store, vid: u32) -> Option<usize> {
|
||||
store
|
||||
.locations
|
||||
.iter()
|
||||
.position(|loc| loc.find_volume(VolumeId(vid)).is_some())
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks.
|
||||
#[test]
|
||||
fn test_add_volume_spreads_collection_over_disks() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "other");
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
let first = location_of(&store, 3);
|
||||
// Leave the disk with the "spread" volume holding fewer volumes overall.
|
||||
let other_on_first = if first == Some(1) { 2 } else { 1 };
|
||||
store
|
||||
.delete_volume(VolumeId(other_on_first), false, false, false)
|
||||
.unwrap();
|
||||
|
||||
add_spread_test_volume(&mut store, 4, "spread");
|
||||
assert_ne!(location_of(&store, 4), first);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection.
|
||||
#[test]
|
||||
fn test_add_volume_ignores_full_volumes_of_collection() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
// Every .dat holds at least its superblock, so all volumes are full.
|
||||
store.volume_size_limit.store(1, Ordering::Relaxed);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "spread");
|
||||
let full_disk = location_of(&store, 1);
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
assert_eq!(location_of(&store, 3), full_disk);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection.
|
||||
#[test]
|
||||
fn test_add_volume_ignores_read_only_volumes_of_collection() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "spread");
|
||||
let writable_disk = location_of(&store, 1);
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
assert_ne!(location_of(&store, 3), writable_disk);
|
||||
store
|
||||
.find_volume(VolumeId(3))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_read_only()
|
||||
.unwrap();
|
||||
|
||||
// The other disk holds more volumes, but its "spread" volume is read-only.
|
||||
add_spread_test_volume(&mut store, 4, "spread");
|
||||
assert_ne!(location_of(&store, 4), writable_disk);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_store_delete_collection() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -3395,7 +3499,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
|
||||
let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap();
|
||||
|
||||
@@ -3719,7 +3823,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
let revision = {
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
v.super_block().compaction_revision
|
||||
@@ -3744,7 +3848,7 @@ mod tests {
|
||||
cookie: Cookie(4),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
|
||||
drop(release_tx);
|
||||
copy.join().unwrap().unwrap();
|
||||
|
||||
@@ -8,7 +8,9 @@ use std::path::Path;
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage::disk_location::{DiskLocation, parse_collection_volume_id_pub};
|
||||
use crate::storage::disk_location::{
|
||||
DiskLocation, parse_collection_volume_id_pub, vif_is_ec_volume,
|
||||
};
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
@@ -34,10 +36,14 @@ impl Store {
|
||||
/// `reconcile_ec_shards_across_disks` so the orphan pass can
|
||||
/// prefer the local idx_directory.
|
||||
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
|
||||
self.mirror_ec_metadata_to_shard_disks_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn mirror_ec_metadata_to_shard_disks_only(&mut self, only: Option<VolumeId>) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
let owners = self.index_ecx_owners_for_mirror();
|
||||
let owners = self.index_ecx_owners_for_mirror(only);
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -52,7 +58,7 @@ impl Store {
|
||||
}
|
||||
let mut mirrors: Vec<Mirror> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_shard_disk_volumes(loc);
|
||||
let orphans = collect_shard_disk_volumes(loc, only);
|
||||
for (key, _shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
continue;
|
||||
@@ -111,7 +117,7 @@ impl Store {
|
||||
// Records both idx_dir (where .ecx was found) and data_dir, so
|
||||
// the mirror can resolve .vif from data_dir even when .ecx lives
|
||||
// in idx_directory.
|
||||
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
|
||||
fn index_ecx_owners_for_mirror(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwner> {
|
||||
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
@@ -131,15 +137,18 @@ impl Store {
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
|
||||
// credible owner — skip it so the scan keeps looking for a
|
||||
// real index on a sibling disk (Go's indexEcxOwners).
|
||||
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
owners
|
||||
.entry(EcKey { collection, vid })
|
||||
.or_insert_with(|| EcxOwner {
|
||||
@@ -207,6 +216,33 @@ fn mirror_sidecars_for_volume(
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<usize> {
|
||||
// A local .vif with no EC config (a source volume's own) would shadow the
|
||||
// owner's once the index is copied next to it; keep mounting across disks.
|
||||
let dst_vif = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ".vif");
|
||||
if path_is_regular_file(&dst_vif)
|
||||
&& !vif_is_ec_volume(&dst_vif)
|
||||
&& (vif_is_ec_volume(&sidecar_dest_path(
|
||||
src_data_dir,
|
||||
src_idx_dir,
|
||||
collection,
|
||||
vid,
|
||||
".vif",
|
||||
)) || vif_is_ec_volume(&sidecar_dest_path(
|
||||
src_idx_dir,
|
||||
src_data_dir,
|
||||
collection,
|
||||
vid,
|
||||
".vif",
|
||||
)))
|
||||
{
|
||||
info!(
|
||||
volume_id = vid.0,
|
||||
collection = %collection,
|
||||
vif = %dst_vif,
|
||||
"local .vif records no EC config; not mirroring EC sidecars",
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
let mut copied = 0usize;
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
|
||||
@@ -268,7 +304,10 @@ fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
|
||||
fn collect_shard_disk_volumes(
|
||||
loc: &DiskLocation,
|
||||
only: Option<VolumeId>,
|
||||
) -> HashMap<EcKey, Vec<String>> {
|
||||
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
return out;
|
||||
@@ -285,13 +324,16 @@ fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>>
|
||||
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
match ent.metadata() {
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
out.entry(EcKey { collection, vid }).or_default().push(name);
|
||||
}
|
||||
out
|
||||
@@ -391,6 +433,222 @@ mod tests {
|
||||
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
|
||||
}
|
||||
|
||||
/// Shards on a disk that still holds the source volume's config-free
|
||||
/// `.vif` must mount with the owner's encode identity: mirroring the index
|
||||
/// next to that `.vif` would make the disk mount through it.
|
||||
#[test]
|
||||
fn mirror_skips_disk_with_config_free_vif() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4122u32;
|
||||
plant_shard(&dir0, collection, vid, 2);
|
||||
plant_shard(&dir1, collection, vid, 0);
|
||||
fs::write(
|
||||
dir0.join(format!("{}_{}.vif", collection, vid)),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_000_000_042,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir1.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
assert!(
|
||||
!dir0.join(format!("{}_{}.ecx", collection, vid)).exists(),
|
||||
".ecx mirrored next to the config-free .vif"
|
||||
);
|
||||
let ec_vol = store.locations[0]
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.expect("dir0 ec volume mounted");
|
||||
assert_eq!(ec_vol.encode_ts_ns, 1_700_000_000_000_000_042);
|
||||
}
|
||||
|
||||
/// With a shared `-dir.idx` every disk sees the `.ecx`, and only the disk
|
||||
/// that received it holds the encode's `.vif`. A disk loaded first that
|
||||
/// still holds the source volume's config-free `.vif` must take that
|
||||
/// identity from its sibling.
|
||||
#[test]
|
||||
fn shared_index_disk_takes_sibling_vif_on_startup() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
let idx = tmp.path().join("idx");
|
||||
for d in [&dir0, &dir1, &idx] {
|
||||
fs::create_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4123u32;
|
||||
plant_shard(&dir0, collection, vid, 2);
|
||||
plant_shard(&dir1, collection, vid, 0);
|
||||
fs::write(
|
||||
dir0.join(format!("{}_{}.vif", collection, vid)),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: 1_700_000_000_000_000_043,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir1.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store.store_dirs = vec![
|
||||
dir0.to_str().unwrap().to_string(),
|
||||
dir1.to_str().unwrap().to_string(),
|
||||
];
|
||||
for dir in [&dir0, &dir1] {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
idx.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for loc in &store.locations {
|
||||
let ec_vol = loc
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.unwrap_or_else(|| panic!("{} ec volume mounted", loc.directory));
|
||||
assert_eq!(
|
||||
ec_vol.encode_ts_ns, 1_700_000_000_000_000_043,
|
||||
"{}",
|
||||
loc.directory
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a store whose first disk still holds the source `.dat`, its
|
||||
/// config-free `.vif` and one 12+4 shard, with the `.ecx` in a shared
|
||||
/// index dir and one sibling encode `.vif` per entry of `sibling_runs`.
|
||||
/// Asserts the source `.dat`, the shard and the `.ecx` survive startup.
|
||||
fn start_with_sibling_encode_vifs(sibling_runs: &[i64]) -> Store {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dirs: Vec<_> = (0..=sibling_runs.len())
|
||||
.map(|i| tmp.path().join(format!("data{}", i)))
|
||||
.collect();
|
||||
let idx = tmp.path().join("idx");
|
||||
for d in dirs.iter().chain([&idx]) {
|
||||
fs::create_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
let collection = "c";
|
||||
let vid = 4124u32;
|
||||
let base0 = dirs[0].join(format!("{}_{}", collection, vid));
|
||||
let dat = format!("{}.dat", base0.display());
|
||||
let shard = format!("{}.ec02", base0.display());
|
||||
fs::File::create(&dat)
|
||||
.unwrap()
|
||||
.set_len(11 * 1024 * 1024)
|
||||
.unwrap();
|
||||
fs::File::create(&shard)
|
||||
.unwrap()
|
||||
.set_len(1024 * 1024)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
format!("{}.vif", base0.display()),
|
||||
r#"{"version":3,"readOnly":true}"#,
|
||||
)
|
||||
.unwrap();
|
||||
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
|
||||
for (dir, run) in dirs[1..].iter().zip(sibling_runs) {
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 12,
|
||||
parity_shards: 4,
|
||||
encode_ts_ns: *run,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
fs::write(
|
||||
dir.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store.store_dirs = dirs
|
||||
.iter()
|
||||
.map(|d| d.to_str().unwrap().to_string())
|
||||
.collect();
|
||||
for dir in &dirs {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
idx.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let ecx = idx.join(format!("{}_{}.ecx", collection, vid));
|
||||
for path in [dat.as_str(), shard.as_str(), ecx.to_str().unwrap()] {
|
||||
assert!(Path::new(path).exists(), "{} removed at startup", path);
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
/// A disk still holding the source `.dat` sizes its shards by the
|
||||
/// sibling's encode `.vif` before the startup check decides whether they
|
||||
/// are leftovers of an interrupted encode.
|
||||
#[test]
|
||||
fn shared_index_startup_validation_uses_sibling_vif_ratio() {
|
||||
let store = start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044]);
|
||||
let ec_vol = store.locations[0]
|
||||
.find_ec_volume(VolumeId(4124))
|
||||
.expect("dir0 ec volume mounted");
|
||||
assert!(ec_vol.has_shard(2), "shard 2 not mounted on dir0");
|
||||
}
|
||||
|
||||
/// Sibling `.vif` files naming different encode runs leave the ratio
|
||||
/// unknown, so the startup check keeps the shards.
|
||||
#[test]
|
||||
fn shared_index_startup_validation_keeps_shards_on_disagreeing_siblings() {
|
||||
start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044, 1_700_000_000_000_000_045]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_preserves_existing_destination_sidecars() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -443,7 +701,7 @@ mod tests {
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
let owners = store.index_ecx_owners_for_mirror();
|
||||
let owners = store.index_ecx_owners_for_mirror(None);
|
||||
let owner = owners
|
||||
.get(&EcKey {
|
||||
collection: collection.to_string(),
|
||||
|
||||
@@ -92,11 +92,15 @@ impl Store {
|
||||
///
|
||||
/// Mirrors `Store.reconcileEcShardsAcrossDisks` in Go.
|
||||
pub fn reconcile_ec_shards_across_disks(&mut self) {
|
||||
self.reconcile_ec_shards_across_disks_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn reconcile_ec_shards_across_disks_only(&mut self, only: Option<VolumeId>) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
|
||||
let owners = self.index_ecx_owners();
|
||||
let owners = self.index_ecx_owners(only);
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -106,7 +110,7 @@ impl Store {
|
||||
// loc.idx_directory instead of the owner disk.
|
||||
let mut to_load: Vec<OrphanShardLoad> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_orphan_ec_shards(loc, loc_idx);
|
||||
let orphans = collect_orphan_ec_shards(loc, loc_idx, only);
|
||||
for (key, shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
warn!(
|
||||
@@ -402,7 +406,7 @@ impl Store {
|
||||
/// matched so downstream loaders point `EcVolume::new` at the
|
||||
/// directory that really has the file. The first owner found wins;
|
||||
/// duplicates across disks are unusual but tolerated.
|
||||
fn index_ecx_owners(&self) -> HashMap<EcKey, EcxOwnerInfo> {
|
||||
fn index_ecx_owners(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwnerInfo> {
|
||||
let mut owners: HashMap<EcKey, EcxOwnerInfo> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
@@ -422,15 +426,18 @@ impl Store {
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
|
||||
// credible owner — skip it so the scan keeps looking for a
|
||||
// real index on a sibling disk (Go's indexEcxOwners).
|
||||
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
let key = EcKey { collection, vid };
|
||||
owners.entry(key).or_insert(EcxOwnerInfo {
|
||||
location: loc_idx,
|
||||
@@ -450,11 +457,18 @@ impl Store {
|
||||
///
|
||||
/// Mirrors `Store.CollectEcVolumesMissingIndex` in Go.
|
||||
pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec<EcVolumeMissingIndex> {
|
||||
let owners = self.index_ecx_owners();
|
||||
self.collect_ec_volumes_missing_index_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_ec_volumes_missing_index_only(
|
||||
&self,
|
||||
only: Option<VolumeId>,
|
||||
) -> Vec<EcVolumeMissingIndex> {
|
||||
let owners = self.index_ecx_owners(only);
|
||||
let mut seen: HashSet<EcKey> = HashSet::new();
|
||||
let mut missing = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) {
|
||||
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
|
||||
if owners.contains_key(&key) || !seen.insert(key.clone()) {
|
||||
continue;
|
||||
}
|
||||
@@ -474,18 +488,24 @@ impl Store {
|
||||
/// the disks that now have a local index, then fall back to the cross-disk
|
||||
/// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go.
|
||||
pub(crate) fn mount_recovered_ec_shards(&mut self) {
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
self.load_orphan_ec_shards_with_local_index();
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
self.mount_recovered_ec_shards_only(None)
|
||||
}
|
||||
|
||||
/// `mount_recovered_ec_shards` restricted to one volume's shards when
|
||||
/// `only` names it.
|
||||
pub(crate) fn mount_recovered_ec_shards_only(&mut self, only: Option<VolumeId>) {
|
||||
self.mirror_ec_metadata_to_shard_disks_only(only);
|
||||
self.load_orphan_ec_shards_with_local_index(only);
|
||||
self.reconcile_ec_shards_across_disks_only(only);
|
||||
}
|
||||
|
||||
/// Mount on-disk EC shards whose `.ecx` index is now present on the same disk.
|
||||
/// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a
|
||||
/// single-disk store recovers once its index has been fetched from a peer.
|
||||
fn load_orphan_ec_shards_with_local_index(&mut self) {
|
||||
fn load_orphan_ec_shards_with_local_index(&mut self, only: Option<VolumeId>) {
|
||||
let mut work: Vec<(usize, EcKey, Vec<ShardId>)> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) {
|
||||
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
|
||||
if !loc.has_ecx_file_on_disk(&key.collection, key.vid) {
|
||||
continue;
|
||||
}
|
||||
@@ -516,6 +536,7 @@ impl Store {
|
||||
fn collect_orphan_ec_shards(
|
||||
loc: &crate::storage::disk_location::DiskLocation,
|
||||
_loc_idx: usize,
|
||||
only: Option<VolumeId>,
|
||||
) -> HashMap<EcKey, Vec<(String, ShardId)>> {
|
||||
let mut orphans: HashMap<EcKey, Vec<(String, ShardId)>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
@@ -533,6 +554,12 @@ fn collect_orphan_ec_shards(
|
||||
let Some(shard_id) = is_ec_shard_extension(ext) else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// Ignore zero-byte shards. Use the DirEntry's metadata so we
|
||||
// don't pay a second stat syscall per file beyond what
|
||||
// read_dir already returned.
|
||||
@@ -540,9 +567,6 @@ fn collect_orphan_ec_shards(
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
// Skip shards that are already registered to an EcVolume.
|
||||
if let Some(ecv) = loc.find_ec_volume(vid)
|
||||
&& ecv.has_shard(shard_id)
|
||||
@@ -655,7 +679,7 @@ mod tests {
|
||||
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
|
||||
write_index_files(&d1, "pics", 7, 10, 4);
|
||||
|
||||
let owners = store.index_ecx_owners();
|
||||
let owners = store.index_ecx_owners(None);
|
||||
let owner = owners
|
||||
.get(&EcKey {
|
||||
collection: "pics".to_string(),
|
||||
@@ -667,7 +691,7 @@ mod tests {
|
||||
|
||||
// A stub with no real index anywhere owns nothing.
|
||||
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
|
||||
assert!(!store.index_ecx_owners().contains_key(&EcKey {
|
||||
assert!(!store.index_ecx_owners(None).contains_key(&EcKey {
|
||||
collection: "pics".to_string(),
|
||||
vid: VolumeId(8),
|
||||
}));
|
||||
@@ -1240,6 +1264,79 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mount_recovered_ec_shards_only_named_volume() {
|
||||
// Recovering one volume mounts only its shards. Another volume's orphan
|
||||
// shards, such as ones an in-flight copy just renamed in, stay unmounted
|
||||
// even when an index for them is already on disk. Mirrors Go
|
||||
// TestMountRecoveredEcShards_OnlyNamedVolume.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
std::fs::create_dir_all(&dir0).unwrap();
|
||||
std::fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "col";
|
||||
let (recovered_vid, other_vid) = (71u32, 72u32);
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for d in [&dir0, &dir1] {
|
||||
store
|
||||
.add_location(
|
||||
d.to_str().unwrap(),
|
||||
d.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for vid in [recovered_vid, other_vid] {
|
||||
write_shard(dir0.to_str().unwrap(), collection, vid, 0);
|
||||
write_shard(dir1.to_str().unwrap(), collection, vid, 1);
|
||||
}
|
||||
write_index_files(dir0.to_str().unwrap(), collection, other_vid, 10, 4);
|
||||
let missing = store.collect_ec_volumes_missing_index();
|
||||
let m = missing
|
||||
.iter()
|
||||
.find(|m| m.vid == VolumeId(recovered_vid))
|
||||
.expect("recovered volume not reported as missing its index");
|
||||
write_index_files(&m.idx_dir, collection, recovered_vid, 10, 4);
|
||||
|
||||
store.mount_recovered_ec_shards_only(Some(VolumeId(recovered_vid)));
|
||||
|
||||
for (i, loc) in store.locations.iter().enumerate() {
|
||||
let ev = loc
|
||||
.find_ec_volume(VolumeId(recovered_vid))
|
||||
.unwrap_or_else(|| panic!("volume {} not mounted on disk{}", recovered_vid, i));
|
||||
assert!(
|
||||
ev.has_shard(i as u8),
|
||||
"shard {}.{} not mounted",
|
||||
recovered_vid,
|
||||
i
|
||||
);
|
||||
assert!(
|
||||
loc.find_ec_volume(VolumeId(other_vid)).is_none(),
|
||||
"volume {} mounted on disk{} by the recovery of volume {}",
|
||||
other_vid,
|
||||
i,
|
||||
recovered_vid,
|
||||
);
|
||||
}
|
||||
let mirrored = format!(
|
||||
"{}/{}_{}.ecx",
|
||||
dir1.to_str().unwrap(),
|
||||
collection,
|
||||
other_vid
|
||||
);
|
||||
assert!(
|
||||
!std::path::Path::new(&mirrored).exists(),
|
||||
"volume {} index mirrored to disk1 by the recovery of volume {}",
|
||||
other_vid,
|
||||
recovered_vid,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_missing_index_excludes_cross_disk_orphan() {
|
||||
// A .ecx merely on a sibling disk is the same-server case the reconcile
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -188,7 +188,7 @@ mod tests {
|
||||
for id in 1..=3 {
|
||||
v.write_needle(&mut needle(id), true, false).unwrap();
|
||||
}
|
||||
v.delete_needle(&mut needle(2)).unwrap();
|
||||
v.delete_needle(&mut needle(2), false).unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
let (want_count, want_deleted) = (v.file_count(), v.deleted_count());
|
||||
drop(v);
|
||||
|
||||
@@ -382,10 +382,13 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 6);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
@@ -417,10 +420,13 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 8);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
|
||||
@@ -505,10 +505,13 @@ mod tests {
|
||||
f.written.insert(*id, n.data.clone());
|
||||
}
|
||||
Op::Del(id) => {
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
})
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
Op::Empty(id) => {
|
||||
|
||||
@@ -110,6 +110,7 @@ fn build_test_state(
|
||||
vacuum_commit_slots: None,
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
|
||||
@@ -3,11 +3,12 @@ use std::collections::HashMap;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions};
|
||||
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
|
||||
use seaweed_worker_core::pb::{
|
||||
ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
|
||||
JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
|
||||
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
RunDetectionRequest,
|
||||
};
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
@@ -15,6 +16,7 @@ use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::purge::{self, Retention};
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
|
||||
pub const JOB_TYPE: &str = "lance_cleanup_versions";
|
||||
@@ -28,7 +30,9 @@ const MAX_MIN_VERSIONS: i64 = 1000;
|
||||
|
||||
/// Lance keeps every version until something removes it. Lance can also do this
|
||||
/// itself through auto-cleanup, so this job is for deployments that would rather
|
||||
/// the cluster owned the policy than each writer.
|
||||
/// the cluster owned the policy than each writer. A table may override the
|
||||
/// retention in its own config, and may ask for a purge (see [`purge`]), which
|
||||
/// this job carries out ahead of the retention window.
|
||||
pub struct CleanupVersionsHandler {
|
||||
namespace_url: String,
|
||||
storage: dataset::Storage,
|
||||
@@ -68,9 +72,8 @@ impl CleanupVersionsHandler {
|
||||
|
||||
/// The oldest version that may be removed while still leaving `min_versions`
|
||||
/// behind, or None when the table has no more than the floor.
|
||||
async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result<Option<u64>> {
|
||||
let mut versions: Vec<u64> = table
|
||||
.dataset
|
||||
async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<u64>> {
|
||||
let mut versions: Vec<u64> = dataset
|
||||
.versions()
|
||||
.await?
|
||||
.iter()
|
||||
@@ -83,6 +86,33 @@ async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Resul
|
||||
Ok(Some(versions[versions.len() - min_versions]))
|
||||
}
|
||||
|
||||
/// The cleanup that keeps `min_versions` and every version younger than
|
||||
/// `retain_hours`, or None when the floor leaves nothing to remove. A purge
|
||||
/// cleans up through it too.
|
||||
pub async fn retention_policy(
|
||||
dataset: &Dataset,
|
||||
retain_hours: i64,
|
||||
min_versions: usize,
|
||||
) -> Result<Option<CleanupPolicy>> {
|
||||
let Some(floor) = version_floor(dataset, min_versions).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(CleanupPolicy {
|
||||
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
|
||||
// should_clean() ANDs its clauses, so a version has to be both older
|
||||
// than the window and below the floor to go.
|
||||
before_version: Some(floor),
|
||||
// Files this dataset cannot account for are left alone: they may
|
||||
// belong to a writer that has not committed yet, and deleting them
|
||||
// would corrupt a commit in flight.
|
||||
delete_unverified: false,
|
||||
// A tagged version is pinned on purpose, so refuse rather than
|
||||
// silently dropping what someone named.
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for CleanupVersionsHandler {
|
||||
fn capability(&self) -> JobTypeCapability {
|
||||
@@ -134,6 +164,12 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
defaults.clone(),
|
||||
)),
|
||||
worker_default_values: defaults,
|
||||
// A purge compacts and retrains, so it gets a build's time.
|
||||
admin_runtime_defaults: Some(AdminRuntimeDefaults {
|
||||
job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
|
||||
execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -181,10 +217,23 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let config = table.dataset.config();
|
||||
let pending_purge = purge::pending(config);
|
||||
let floor = Retention::for_table(
|
||||
config,
|
||||
Retention {
|
||||
retain_hours: DEFAULT_RETAIN_HOURS,
|
||||
min_versions,
|
||||
},
|
||||
MAX_RETAIN_HOURS,
|
||||
MAX_MIN_VERSIONS,
|
||||
)
|
||||
.min_versions;
|
||||
// Age is decided at execution against the retention window; a table
|
||||
// at or under the floor cannot lose a version whatever its age, so
|
||||
// proposing one would only produce a job with nothing to do.
|
||||
if stats.total_versions <= min_versions {
|
||||
// proposing one would only produce a job with nothing to do. A
|
||||
// requested purge ignores the floor: it keeps only the latest.
|
||||
if pending_purge.is_none() && stats.total_versions <= floor {
|
||||
continue;
|
||||
}
|
||||
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
|
||||
@@ -193,12 +242,21 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
proposal_id: format!("{JOB_TYPE}:{encoded}"),
|
||||
dedupe_key: format!("{JOB_TYPE}:{encoded}"),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
summary: format!("Clean up {encoded} ({} versions)", stats.total_versions),
|
||||
detail: format!(
|
||||
"{} versions retained, above the {min_versions} floor; \
|
||||
those outside the retention window can go",
|
||||
stats.total_versions
|
||||
),
|
||||
summary: match &pending_purge {
|
||||
Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions),
|
||||
None => format!("Clean up {encoded} ({} versions)", stats.total_versions),
|
||||
},
|
||||
detail: match &pending_purge {
|
||||
Some(token) => format!(
|
||||
"purge requested at {token}: rewrite fragments with deletions \
|
||||
and keep only the latest version"
|
||||
),
|
||||
None => format!(
|
||||
"{} versions retained, above the {floor} floor; \
|
||||
those outside the retention window can go",
|
||||
stats.total_versions
|
||||
),
|
||||
},
|
||||
parameters,
|
||||
..Default::default()
|
||||
});
|
||||
@@ -254,14 +312,33 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
) as usize;
|
||||
|
||||
let client = self.client();
|
||||
let table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let mut table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let before = table.stats().await?;
|
||||
|
||||
if let Some(token) = purge::pending(table.dataset.config()) {
|
||||
return self
|
||||
.execute_purge(request, sender, &job.job_id, &mut table, &token)
|
||||
.await;
|
||||
}
|
||||
let Retention {
|
||||
retain_hours,
|
||||
min_versions,
|
||||
} = Retention::for_table(
|
||||
table.dataset.config(),
|
||||
Retention {
|
||||
retain_hours,
|
||||
min_versions,
|
||||
},
|
||||
MAX_RETAIN_HOURS,
|
||||
MAX_MIN_VERSIONS,
|
||||
);
|
||||
|
||||
// The floor is a promise about how much history survives, so it has to
|
||||
// be applied here and not only when the job was proposed: by the time it
|
||||
// runs, versions may have aged past the retention window, and age alone
|
||||
// would take the table below what the operator asked to keep.
|
||||
let Some(floor) = version_floor(&table, min_versions).await? else {
|
||||
let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await?
|
||||
else {
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
@@ -288,20 +365,6 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
let policy = CleanupPolicy {
|
||||
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
|
||||
// should_clean() ANDs its clauses, so a version has to be both older
|
||||
// than the window and below the floor to go.
|
||||
before_version: Some(floor),
|
||||
// Files this dataset cannot account for are left alone: they may
|
||||
// belong to a writer that has not committed yet, and deleting them
|
||||
// would corrupt a commit in flight.
|
||||
delete_unverified: false,
|
||||
// A tagged version is pinned on purpose, so refuse rather than
|
||||
// silently dropping what someone named.
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
};
|
||||
let stats = cleanup_old_versions(&table.dataset, policy)
|
||||
.await
|
||||
.with_context(|| format!("clean up versions of {}", table.location))?;
|
||||
@@ -343,3 +406,73 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl CleanupVersionsHandler {
|
||||
async fn execute_purge(
|
||||
&self,
|
||||
request: &ExecuteJobRequest,
|
||||
sender: &dyn ExecutionSender,
|
||||
job_id: &str,
|
||||
table: &mut dataset::OpenTable,
|
||||
token: &str,
|
||||
) -> Result<()> {
|
||||
let stats = purge::purge(&mut table.dataset, token)
|
||||
.await
|
||||
.with_context(|| format!("purge {}", table.location))?;
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.versions_removed.inc_by(stats.old_versions);
|
||||
counters.bytes_reclaimed.inc_by(stats.bytes_removed);
|
||||
}
|
||||
let mut output: HashMap<String, ConfigValue> = HashMap::new();
|
||||
output.insert(
|
||||
"fragments_rewritten".to_string(),
|
||||
int_value(stats.fragments_rewritten as i64),
|
||||
);
|
||||
output.insert(
|
||||
"old_versions_removed".to_string(),
|
||||
int_value(stats.old_versions as i64),
|
||||
);
|
||||
output.insert(
|
||||
"bytes_removed".to_string(),
|
||||
int_value(stats.bytes_removed as i64),
|
||||
);
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job_id.to_string(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
result: Some(JobResult {
|
||||
output_values: output,
|
||||
summary: if stats.served {
|
||||
format!(
|
||||
"purged: rewrote {} fragments, removed {} versions and {} bytes",
|
||||
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"purge pending: rewrote {} fragments, removed {} versions and {} bytes; \
|
||||
versions from before the rebuild are still retained",
|
||||
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
|
||||
)
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_purge_gets_a_builds_execution_timeout() {
|
||||
let descriptor = CleanupVersionsHandler::new(String::new()).descriptor();
|
||||
let defaults = descriptor.admin_runtime_defaults.unwrap();
|
||||
assert_eq!(defaults.execution_timeout_seconds, 3600);
|
||||
assert_eq!(defaults.job_type_max_runtime_seconds, 3600);
|
||||
assert!(!defaults.enabled);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
pub mod cleanup;
|
||||
pub mod compact;
|
||||
pub mod indices;
|
||||
pub mod purge;
|
||||
pub mod sort;
|
||||
pub mod vector_index;
|
||||
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
//! Per-table retention for version cleanup, and purges: compaction that
|
||||
//! rewrites every fragment with deletions followed by a cleanup that keeps
|
||||
//! only the latest version, so the bytes of deleted rows leave storage now
|
||||
//! rather than after the retention window.
|
||||
//!
|
||||
//! Both are set in the table's own Lance config, so any client that can write
|
||||
//! the table can set them and they travel with it:
|
||||
//!
|
||||
//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions`
|
||||
//! override the worker's retention for this table.
|
||||
//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`]
|
||||
//! writes the current time); a purge is pending while it differs from
|
||||
//! `seaweedfs.purge.completed`, which the purge sets to the token it served.
|
||||
//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use chrono::Utc;
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions};
|
||||
use lance::dataset::optimize::{CompactionOptions, compact_files};
|
||||
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
|
||||
use lance_index::metrics::NoOpMetricsCollector;
|
||||
use lance_index::optimize::OptimizeOptions;
|
||||
use lance_index::{IndexType, is_system_index};
|
||||
use lance_table::format::IndexMetadata;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::jobs::cleanup::retention_policy;
|
||||
|
||||
pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours";
|
||||
pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions";
|
||||
pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested";
|
||||
pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed";
|
||||
pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt";
|
||||
|
||||
/// The retention cleanup applies to one table: the worker's, unless the
|
||||
/// table's config overrides a value with one inside the worker's range.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Retention {
|
||||
pub retain_hours: i64,
|
||||
pub min_versions: usize,
|
||||
}
|
||||
|
||||
impl Retention {
|
||||
pub fn for_table(
|
||||
config: &HashMap<String, String>,
|
||||
worker: Retention,
|
||||
max_retain_hours: i64,
|
||||
max_min_versions: i64,
|
||||
) -> Retention {
|
||||
let read = |key: &str, min: i64, max: i64| -> Option<i64> {
|
||||
let raw = config.get(key)?;
|
||||
match raw.trim().parse::<i64>() {
|
||||
Ok(value) if (min..=max).contains(&value) => Some(value),
|
||||
_ => {
|
||||
warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}");
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
Retention {
|
||||
retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours)
|
||||
.unwrap_or(worker.retain_hours),
|
||||
min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions)
|
||||
.map(|v| v as usize)
|
||||
.unwrap_or(worker.min_versions),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The request token of a purge the table is waiting for, if any.
|
||||
pub fn pending(config: &HashMap<String, String>) -> Option<String> {
|
||||
let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?;
|
||||
if requested.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match config.get(PURGE_COMPLETED_KEY) {
|
||||
Some(completed) if completed.trim() == requested => None,
|
||||
_ => Some(requested.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a purge request on the table and returns its token. The next
|
||||
/// cleanup detection proposes the purge.
|
||||
pub async fn request_purge(dataset: &mut Dataset) -> Result<String> {
|
||||
let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
|
||||
dataset
|
||||
.update_config([(PURGE_REQUESTED_KEY, token.as_str())])
|
||||
.await
|
||||
.context("record purge request")?;
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct PurgeStats {
|
||||
pub fragments_rewritten: usize,
|
||||
pub old_versions: u64,
|
||||
pub bytes_removed: u64,
|
||||
/// False while versions from before the rebuild remain.
|
||||
pub served: bool,
|
||||
}
|
||||
|
||||
/// Rewrites every fragment that has deleted rows and rebuilds the indices from
|
||||
/// the rows that remain, then removes the versions from before that through
|
||||
/// the cleanup job's retention policy with only the latest version kept, and
|
||||
/// marks `token` served once none of them is left. The rebuild is needed
|
||||
/// because compaction only remaps an index: an IVF_FLAT index keeps the
|
||||
/// deleted vector's values in its partitions under a remapped row address. It
|
||||
/// runs once per request, so a purge that has to wait for its cleanup (a
|
||||
/// version pinned by a tag, a query lease or one someone named, fails it)
|
||||
/// retries only the cleanup. Files no version references stay while Lance
|
||||
/// cannot tell them from a writer's uncommitted ones, which is up to seven
|
||||
/// days for an abandoned write.
|
||||
pub async fn purge(dataset: &mut Dataset, token: &str) -> Result<PurgeStats> {
|
||||
let fragments_rewritten = rebuild(dataset, token).await?;
|
||||
let cleanup = match retention_policy(dataset, 0, 1).await? {
|
||||
Some(policy) => cleanup_old_versions(dataset, policy)
|
||||
.await
|
||||
.context("remove versions older than the purge")?,
|
||||
None => RemovalStats::default(),
|
||||
};
|
||||
let served = finish(dataset, token).await?;
|
||||
Ok(PurgeStats {
|
||||
fragments_rewritten,
|
||||
old_versions: cleanup.old_versions,
|
||||
bytes_removed: cleanup.bytes_removed,
|
||||
served,
|
||||
})
|
||||
}
|
||||
|
||||
/// Rewrites the fragments with deletions and rebuilds the indices for
|
||||
/// `token`, unless that was done already, and returns how many fragments it
|
||||
/// rewrote.
|
||||
pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result<usize> {
|
||||
if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) {
|
||||
return Ok(0);
|
||||
}
|
||||
let vector_indices = vector_index_names(dataset).await?;
|
||||
let options = CompactionOptions {
|
||||
materialize_deletions: true,
|
||||
// Zero materializes every fragment with any deletion.
|
||||
materialize_deletions_threshold: 0.0,
|
||||
..Default::default()
|
||||
};
|
||||
let compaction = compact_files(dataset, options, None)
|
||||
.await
|
||||
.context("rewrite fragments with deletions")?;
|
||||
dataset.checkout_latest().await?;
|
||||
if !vector_indices.is_empty() {
|
||||
let mut options = OptimizeOptions::retrain();
|
||||
options.index_names = Some(vector_indices);
|
||||
dataset
|
||||
.optimize_indices(&options)
|
||||
.await
|
||||
.context("rebuild indices without the deleted rows")?;
|
||||
}
|
||||
rebuild_scalar_indices(dataset).await?;
|
||||
dataset.checkout_latest().await?;
|
||||
dataset
|
||||
.update_config([(PURGE_REBUILT_KEY, token)])
|
||||
.await
|
||||
.context("record purge rebuild")?;
|
||||
Ok(compaction.fragments_removed)
|
||||
}
|
||||
|
||||
/// Marks `token` served when the oldest version left is one the rebuild for
|
||||
/// it wrote, so no version from before the rebuild remains.
|
||||
pub async fn finish(dataset: &mut Dataset, token: &str) -> Result<bool> {
|
||||
let oldest = dataset
|
||||
.versions()
|
||||
.await?
|
||||
.iter()
|
||||
.map(|v| v.version)
|
||||
.min()
|
||||
.context("the table has no versions")?;
|
||||
let oldest = dataset.checkout_version(oldest).await?;
|
||||
if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) {
|
||||
return Ok(false);
|
||||
}
|
||||
dataset
|
||||
.update_config([(PURGE_COMPLETED_KEY, token)])
|
||||
.await
|
||||
.context("record purge completion")?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Names of the vector indices. A retrain builds each name over the whole
|
||||
/// table, so a column whose index is sharded over several names is refused:
|
||||
/// its shards would overlap.
|
||||
async fn vector_index_names(dataset: &Dataset) -> Result<Vec<String>> {
|
||||
let mut by_column: BTreeMap<Vec<i32>, BTreeSet<String>> = BTreeMap::new();
|
||||
for index in dataset
|
||||
.load_indices()
|
||||
.await?
|
||||
.iter()
|
||||
.filter(|i| is_vector(i))
|
||||
{
|
||||
by_column
|
||||
.entry(index.fields.clone())
|
||||
.or_default()
|
||||
.insert(index.name.clone());
|
||||
}
|
||||
let mut names = Vec::new();
|
||||
for shards in by_column.into_values() {
|
||||
if shards.len() > 1 {
|
||||
bail!(
|
||||
"the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table"
|
||||
);
|
||||
}
|
||||
names.extend(shards);
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
/// Builds every scalar index again from the rows that remain. Compaction only
|
||||
/// remaps them: a bitmap index keeps a deleted key, and under stable row ids
|
||||
/// no scalar index is rewritten at all.
|
||||
async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> {
|
||||
let indices = dataset.load_indices().await?;
|
||||
let mut rebuilt = BTreeSet::new();
|
||||
for index in indices.iter() {
|
||||
if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) {
|
||||
continue;
|
||||
}
|
||||
let [field] = index.fields[..] else {
|
||||
bail!("index {} covers {} columns", index.name, index.fields.len());
|
||||
};
|
||||
let column = dataset.schema().field_path(field)?;
|
||||
let params = dataset
|
||||
.open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector)
|
||||
.await?
|
||||
.derive_index_params()?;
|
||||
dataset
|
||||
.create_index(
|
||||
&[column.as_str()],
|
||||
IndexType::Scalar,
|
||||
Some(index.name.clone()),
|
||||
¶ms,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("rebuild index {}", index.name))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_vector(index: &IndexMetadata) -> bool {
|
||||
index
|
||||
.index_details
|
||||
.as_ref()
|
||||
.is_some_and(|details| details.type_url.ends_with("VectorIndexDetails"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config(pairs: &[(&str, &str)]) -> HashMap<String, String> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
const WORKER: Retention = Retention {
|
||||
retain_hours: 168,
|
||||
min_versions: 5,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn table_retention_overrides_within_range() {
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[]), WORKER, 8760, 1000),
|
||||
WORKER
|
||||
);
|
||||
assert_eq!(
|
||||
Retention::for_table(
|
||||
&config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]),
|
||||
WORKER,
|
||||
8760,
|
||||
1000
|
||||
),
|
||||
Retention {
|
||||
retain_hours: 0,
|
||||
min_versions: 1
|
||||
}
|
||||
);
|
||||
for bad in ["-1", "8761", "x", ""] {
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000)
|
||||
.retain_hours,
|
||||
168,
|
||||
"{bad:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000)
|
||||
.min_versions,
|
||||
5
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purge_is_pending_until_its_token_is_served() {
|
||||
assert_eq!(pending(&config(&[])), None);
|
||||
assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None);
|
||||
assert_eq!(
|
||||
pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])),
|
||||
Some("t1".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
pending(&config(&[
|
||||
(PURGE_REQUESTED_KEY, "t2"),
|
||||
(PURGE_COMPLETED_KEY, "t1")
|
||||
])),
|
||||
Some("t2".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
pending(&config(&[
|
||||
(PURGE_REQUESTED_KEY, "t2"),
|
||||
(PURGE_COMPLETED_KEY, "t2")
|
||||
])),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
//! A purge removes a deleted vector's bytes from storage: after it, no file
|
||||
//! under the table holds the vector, its key or its metadata. Runs on a local
|
||||
//! table, so it needs no cluster.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{CleanupPolicy, RemovalStats, cleanup_old_versions};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance::index::vector::VectorIndexParams;
|
||||
use lance_index::IndexType;
|
||||
use lance_linalg::distance::DistanceType;
|
||||
use weed_lance_worker::jobs::purge;
|
||||
|
||||
const DIM: i32 = 8;
|
||||
const ROWS: usize = 300;
|
||||
const MARKER_ROW: usize = 137;
|
||||
const KEY_MARKER: &str = "purge-key-5f0c2e9a7d41b3c8";
|
||||
const METADATA_MARKER: &str = "purge-metadata-a93e1d5b77c2f046";
|
||||
|
||||
fn marker_vector() -> Vec<f32> {
|
||||
// Values no other row has, so their bytes identify the row.
|
||||
(0..DIM).map(|i| 31_337.125 + i as f32 * 0.5).collect()
|
||||
}
|
||||
|
||||
fn batch() -> (Arc<Schema>, RecordBatch) {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("key", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
|
||||
false,
|
||||
),
|
||||
Field::new("metadata", DataType::Utf8, true),
|
||||
]));
|
||||
let keys: Vec<String> = (0..ROWS)
|
||||
.map(|i| {
|
||||
if i == MARKER_ROW {
|
||||
KEY_MARKER.to_string()
|
||||
} else {
|
||||
format!("k{i}")
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let metadata: Vec<String> = (0..ROWS)
|
||||
.map(|i| {
|
||||
if i == MARKER_ROW {
|
||||
format!("{{\"note\":\"{METADATA_MARKER}\"}}")
|
||||
} else {
|
||||
format!("{{\"n\":{i}}}")
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
(0..ROWS).map(|i| {
|
||||
let values: Vec<f32> = if i == MARKER_ROW {
|
||||
marker_vector()
|
||||
} else {
|
||||
(0..DIM)
|
||||
.map(|d| ((i * 7 + d as usize) % 97) as f32)
|
||||
.collect()
|
||||
};
|
||||
Some(values.into_iter().map(Some).collect::<Vec<_>>())
|
||||
}),
|
||||
DIM,
|
||||
);
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(StringArray::from(keys)),
|
||||
Arc::new(vectors),
|
||||
Arc::new(StringArray::from(metadata)),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
(schema, batch)
|
||||
}
|
||||
|
||||
fn vector_bytes() -> Vec<u8> {
|
||||
marker_vector()
|
||||
.iter()
|
||||
.flat_map(|v| v.to_le_bytes())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Each marker found under `dir`, as "<marker> in <file>".
|
||||
fn markers_in(dir: &Path) -> Vec<String> {
|
||||
let vector = vector_bytes();
|
||||
let mut found = Vec::new();
|
||||
let mut stack = vec![dir.to_path_buf()];
|
||||
while let Some(path) = stack.pop() {
|
||||
for entry in std::fs::read_dir(&path).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_dir() {
|
||||
stack.push(entry.path());
|
||||
continue;
|
||||
}
|
||||
let bytes = std::fs::read(entry.path()).unwrap();
|
||||
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
|
||||
for (name, needle) in [
|
||||
("vector", vector.as_slice()),
|
||||
("key", KEY_MARKER.as_bytes()),
|
||||
("metadata", METADATA_MARKER.as_bytes()),
|
||||
] {
|
||||
if has(needle) {
|
||||
let file = entry.path();
|
||||
let file = file.strip_prefix(dir).unwrap_or(&file);
|
||||
found.push(format!("{name} in {}", file.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
found.sort_unstable();
|
||||
found
|
||||
}
|
||||
|
||||
/// A purge whose cleanup nothing holds back, so only the latest version
|
||||
/// stays. [`purge::purge`] cleans up through the cleanup job's retention
|
||||
/// policy, which a deployment may have keep more.
|
||||
async fn purge_to_latest(
|
||||
dataset: &mut Dataset,
|
||||
token: &str,
|
||||
) -> anyhow::Result<(usize, RemovalStats, bool)> {
|
||||
let rewritten = purge::rebuild(dataset, token).await?;
|
||||
let policy = CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now()),
|
||||
before_version: Some(dataset.latest_version_id().await?),
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
};
|
||||
let removed = cleanup_old_versions(dataset, policy).await?;
|
||||
Ok((rewritten, removed, purge::finish(dataset, token).await?))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_removes_a_deleted_vectors_bytes() {
|
||||
purge_leaves_no_trace(false).await;
|
||||
}
|
||||
|
||||
/// Under stable row ids compaction leaves scalar indices as they are, so only
|
||||
/// a rebuild takes the key out of them.
|
||||
#[tokio::test]
|
||||
async fn purge_removes_a_deleted_vectors_bytes_under_stable_row_ids() {
|
||||
purge_leaves_no_trace(true).await;
|
||||
}
|
||||
|
||||
async fn purge_leaves_no_trace(stable_row_ids: bool) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let params = lance::dataset::WriteParams {
|
||||
enable_stable_row_ids: stable_row_ids,
|
||||
..Default::default()
|
||||
};
|
||||
let mut dataset = Dataset::write(
|
||||
RecordBatchIterator::new(vec![Ok(batch)], schema),
|
||||
uri,
|
||||
Some(params),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
Some("vector_idx".to_string()),
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["key"],
|
||||
IndexType::BTree,
|
||||
Some("key_idx".to_string()),
|
||||
&lance_index::scalar::ScalarIndexParams::default(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// A remapped bitmap index still lists the deleted row's value.
|
||||
dataset
|
||||
.create_index(
|
||||
&["metadata"],
|
||||
IndexType::Bitmap,
|
||||
Some("metadata_idx".to_string()),
|
||||
&lance_index::scalar::ScalarIndexParams::for_builtin(
|
||||
lance_index::scalar::BuiltinIndexType::Bitmap,
|
||||
),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
|
||||
let before = markers_in(dir.path());
|
||||
assert!(
|
||||
before.iter().any(|m| m.starts_with("vector ")),
|
||||
"the vector's bytes should be found before deletion, found {before:?}"
|
||||
);
|
||||
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(dataset.count_rows(None).await.unwrap(), ROWS - 1);
|
||||
|
||||
// A deletion alone, and the default retention cleanup, leave the bytes:
|
||||
// the fragment still holds the row and older versions still reference it.
|
||||
cleanup_old_versions(
|
||||
&dataset,
|
||||
CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now() - chrono::Duration::hours(168)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
markers_in(dir.path())
|
||||
.iter()
|
||||
.any(|m| m.starts_with("vector ")),
|
||||
"a deleted vector is expected to stay until purged"
|
||||
);
|
||||
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert!(rewritten >= 1);
|
||||
assert!(removed.old_versions >= 1, "{removed:?}");
|
||||
|
||||
assert_eq!(
|
||||
markers_in(dir.path()),
|
||||
Vec::<String>::new(),
|
||||
"deleted vector's bytes remain in storage after the purge"
|
||||
);
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
|
||||
// The table is intact: every other row, and the index, still serve.
|
||||
let reopened = Dataset::open(uri).await.unwrap();
|
||||
assert_eq!(reopened.count_rows(None).await.unwrap(), ROWS - 1);
|
||||
assert_eq!(purge::pending(reopened.config()), None);
|
||||
assert_eq!(reopened.load_indices().await.unwrap().len(), 3);
|
||||
}
|
||||
|
||||
/// A version a query lease (a tag) pins still holds the deleted row, so the
|
||||
/// purge must fail and leave the request pending rather than report success.
|
||||
/// Once the tag goes, the next purge only cleans up: the rebuild is not done
|
||||
/// again.
|
||||
#[tokio::test]
|
||||
async fn purge_refuses_while_a_tag_pins_old_data() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
Some("vector_idx".to_string()),
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let pinned = dataset.version().version;
|
||||
dataset
|
||||
.tags()
|
||||
.create(
|
||||
"weed-vector-query-lease",
|
||||
lance::dataset::refs::Ref::Version(None, Some(pinned)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert!(purge_to_latest(&mut dataset, &token).await.is_err());
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
|
||||
let index = |dataset: &Dataset| {
|
||||
let dataset = dataset.clone();
|
||||
async move { dataset.load_indices().await.unwrap()[0].uuid }
|
||||
};
|
||||
let rebuilt = index(&dataset).await;
|
||||
dataset
|
||||
.tags()
|
||||
.delete("weed-vector-query-lease")
|
||||
.await
|
||||
.unwrap();
|
||||
let (rewritten, _, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert_eq!(rewritten, 0);
|
||||
assert_eq!(
|
||||
index(&dataset).await,
|
||||
rebuilt,
|
||||
"the index was rebuilt again"
|
||||
);
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
assert_eq!(markers_in(dir.path()), Vec::<String>::new());
|
||||
}
|
||||
|
||||
/// The request is served only once no version from before the rebuild is
|
||||
/// left, whatever kept one.
|
||||
#[tokio::test]
|
||||
async fn purge_is_served_once_older_versions_are_gone() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert_eq!(purge::rebuild(&mut dataset, &token).await.unwrap(), 1);
|
||||
assert!(!purge::finish(&mut dataset, &token).await.unwrap());
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
|
||||
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert_eq!(rewritten, 0);
|
||||
assert!(removed.old_versions >= 1, "{removed:?}");
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
}
|
||||
|
||||
/// A retrain builds every index name over the whole table, so shards of one
|
||||
/// vector index would end up overlapping: the purge refuses before touching
|
||||
/// the table and leaves the request pending.
|
||||
#[tokio::test]
|
||||
async fn purge_refuses_a_sharded_vector_index() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let params = lance::dataset::WriteParams {
|
||||
max_rows_per_file: ROWS / 2,
|
||||
..Default::default()
|
||||
};
|
||||
let mut dataset = Dataset::write(
|
||||
RecordBatchIterator::new(vec![Ok(batch)], schema),
|
||||
uri,
|
||||
Some(params),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let fragments: Vec<u32> = dataset
|
||||
.get_fragments()
|
||||
.iter()
|
||||
.map(|f| f.id() as u32)
|
||||
.collect();
|
||||
assert_eq!(fragments.len(), 2);
|
||||
for (shard, fragment) in fragments.iter().enumerate() {
|
||||
dataset
|
||||
.create_index_builder(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
)
|
||||
.name(format!("shard-{shard}"))
|
||||
.fragments(vec![*fragment])
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
}
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
let version = dataset.version().version;
|
||||
|
||||
let err = purge::purge(&mut dataset, &token).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("sharded"), "{err:#}");
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(dataset.version().version, version, "the table was changed");
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token));
|
||||
}
|
||||
@@ -13,9 +13,9 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.22.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/procfs v0.23.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/seaweedfs/seaweedfs => ../..
|
||||
+12
-16
@@ -2,35 +2,31 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
|
||||
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
|
||||
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/procfs v0.23.0 h1:ad+S6FTkzwaMgacKLM5h6bNKExS+DMH0YiUwXItH/o4=
|
||||
github.com/prometheus/procfs v0.23.0/go.mod h1:3AVXO0FqLafyYFU/V/IQRZlC8Rgqz8Dm/x6pKLK+RIk=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -74,9 +74,15 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
|
||||
|
||||
if limit > 0 && len(series.Versions) > limit {
|
||||
// Old releases are a long tail of one-cluster bands; keep the versions
|
||||
// most of the fleet is on now and sum the rest into one band.
|
||||
// most of the fleet is on now and sum the rest into one band. The
|
||||
// newest release is kept too, while it is still too small to rank, or
|
||||
// clusters upgrading to it look like they left the named versions.
|
||||
newest := newestRelease(series.Versions, last)
|
||||
ranked := append([]VersionCounts(nil), series.Versions...)
|
||||
sort.Slice(ranked, func(i, j int) bool {
|
||||
if (ranked[i].Version == newest) != (ranked[j].Version == newest) {
|
||||
return ranked[i].Version == newest
|
||||
}
|
||||
if ranked[i].Clusters[last] != ranked[j].Clusters[last] {
|
||||
return ranked[i].Clusters[last] > ranked[j].Clusters[last]
|
||||
}
|
||||
@@ -104,6 +110,21 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
|
||||
return series
|
||||
}
|
||||
|
||||
// newestRelease is the newest plain release, without a build suffix, that
|
||||
// clusters run on the last day.
|
||||
func newestRelease(versions []VersionCounts, last int) string {
|
||||
newest := ""
|
||||
for _, v := range versions {
|
||||
if _, suffix, ok := versionParts(v.Version); !ok || suffix != "" || v.Clusters[last] == 0 {
|
||||
continue
|
||||
}
|
||||
if newest == "" || versionLess(newest, v.Version) {
|
||||
newest = v.Version
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// versionedHistories drops the samples recorded before the reported version was
|
||||
// kept in history, so the version series spans the days it actually knows a
|
||||
// version for instead of climbing out of a run of blank days.
|
||||
|
||||
@@ -93,6 +93,32 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionSeriesKeepsNewestRelease(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
|
||||
seedSamples(s, "a", HistorySample{Version: "4.40"}, -2, -1, 0)
|
||||
seedSamples(s, "b", HistorySample{Version: "4.40"}, -2, -1, 0)
|
||||
seedSamples(s, "c", HistorySample{Version: "4.41"}, -2, -1, 0)
|
||||
seedSamples(s, "d", HistorySample{Version: "4.41"}, -2, -1, 0)
|
||||
// Upgraded to the newest release, which is still the smallest band.
|
||||
seedSamples(s, "e", HistorySample{Version: "4.41"}, -2)
|
||||
seedSamples(s, "e", HistorySample{Version: "4.42"}, -1, 0)
|
||||
// A suffixed build newer than every release does not take its place.
|
||||
seedSamples(s, "f", HistorySample{Version: "4.43-dev"}, -2, -1, 0)
|
||||
|
||||
series := s.GetVersionSeries(3, 2)
|
||||
var kept []string
|
||||
for _, v := range series.Versions {
|
||||
kept = append(kept, v.Version)
|
||||
}
|
||||
if len(kept) != 2 || kept[0] != "4.41" || kept[1] != "4.42" {
|
||||
t.Fatalf("kept = %v, want 4.41 and the newest release 4.42", kept)
|
||||
}
|
||||
if series.Other == nil || series.Other.Count != 2 {
|
||||
t.Fatalf("other = %+v, want 4.40 and 4.43-dev", series.Other)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionLess(t *testing.T) {
|
||||
// Ordered oldest to newest; every pair must compare in this order.
|
||||
ordered := []string{"unknown", "3.97", "4.02", "4.30", "4.30-enterprise", "9.99", "10.02"}
|
||||
|
||||
@@ -28,11 +28,13 @@ import (
|
||||
const filerGroup = "fuse-dlm-test"
|
||||
|
||||
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
|
||||
// mounts for testing DLM-based cross-mount write coordination.
|
||||
// mounts for testing DLM-based cross-mount write coordination, and what two
|
||||
// mounts do without it.
|
||||
type dlmTestCluster struct {
|
||||
t testing.TB
|
||||
baseDir string
|
||||
weedBinary string
|
||||
dlm bool // the mounts run with -dlm
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
@@ -52,6 +54,11 @@ type dlmTestCluster struct {
|
||||
}
|
||||
|
||||
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
|
||||
return startTestCluster(t, true)
|
||||
}
|
||||
|
||||
// startTestCluster starts the cluster; dlm says whether the mounts run with -dlm.
|
||||
func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
|
||||
binary := findWeedBinary()
|
||||
if binary == "" {
|
||||
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
|
||||
@@ -64,6 +71,7 @@ func startDLMTestCluster(t testing.TB) *dlmTestCluster {
|
||||
t: t,
|
||||
baseDir: baseDir,
|
||||
weedBinary: binary,
|
||||
dlm: dlm,
|
||||
}
|
||||
// Register cleanup early so processes are stopped even if a require fails below.
|
||||
t.Cleanup(c.Stop)
|
||||
@@ -224,17 +232,20 @@ func (c *dlmTestCluster) startMount(idx int, configDir string) error {
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
args := []string{
|
||||
"-logdir=" + filepath.Join(c.baseDir, "logs"),
|
||||
"mount",
|
||||
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir="+c.mountPoints[idx],
|
||||
"-filer=" + c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir=" + c.mountPoints[idx],
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
"-cacheDir="+cacheDir,
|
||||
"-dlm",
|
||||
)
|
||||
"-cacheDir=" + cacheDir,
|
||||
}
|
||||
if c.dlm {
|
||||
args = append(args, "-dlm")
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary, args...)
|
||||
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// TestStageSameFilesWithoutDLM has two mounts without -dlm stage the same
|
||||
// files into one directory at the same time, the way two jobs extract the same
|
||||
// archive into a shared directory: a file that exists is skipped, otherwise it
|
||||
// is created, written and closed, then chmod'ed and given its mtime. Both
|
||||
// mounts often create the same file, and each then changes the attributes of
|
||||
// the copy it has cached.
|
||||
//
|
||||
// Whatever either mount is told, no entry may point at a chunk that is gone
|
||||
// from the volume server, and every file must read back whole. An update sent
|
||||
// from a cached copy after the other mount rewrote the file puts back the
|
||||
// chunks the rewrite dropped, which the filer deletes; reads of such a file
|
||||
// fail once the volume is vacuumed.
|
||||
func TestStageSameFilesWithoutDLM(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping FUSE integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startTestCluster(t, false)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const (
|
||||
files = 400
|
||||
dir = "staged"
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
names := make([]string, files)
|
||||
content := make(map[string][]byte, files)
|
||||
for i := range names {
|
||||
names[i] = fmt.Sprintf("img-%04d.jpg", i)
|
||||
data := make([]byte, 1024+rng.Intn(63*1024))
|
||||
rng.Read(data)
|
||||
content[names[i]] = data
|
||||
}
|
||||
mtime := time.Date(2026, 10, 26, 4, 0, 0, 0, time.UTC)
|
||||
|
||||
// What the losing mount is told depends on the version (nothing, EIO,
|
||||
// EAGAIN); the files are what this test checks.
|
||||
failed := make([][]string, 2)
|
||||
setupErr := make([]error, 2)
|
||||
wrote := [2][]bool{make([]bool, files), make([]bool, files)}
|
||||
var ready, wg sync.WaitGroup
|
||||
ready.Add(2)
|
||||
for m := 0; m < 2; m++ {
|
||||
wg.Add(1)
|
||||
go func(m int) {
|
||||
defer wg.Done()
|
||||
stage := filepath.Join(cluster.mountPoints[m], dir)
|
||||
setupErr[m] = os.MkdirAll(stage, 0755)
|
||||
ready.Done()
|
||||
ready.Wait()
|
||||
if setupErr[m] != nil {
|
||||
return
|
||||
}
|
||||
for i, name := range names {
|
||||
p := filepath.Join(stage, name)
|
||||
if _, err := os.Lstat(p); err == nil {
|
||||
continue
|
||||
}
|
||||
wrote[m][i] = true
|
||||
if err := os.WriteFile(p, content[name], 0644); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
continue
|
||||
}
|
||||
if err := os.Chmod(p, 0640); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
continue
|
||||
}
|
||||
if err := os.Chtimes(p, mtime, mtime); err != nil {
|
||||
failed[m] = append(failed[m], err.Error())
|
||||
}
|
||||
}
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
for m, f := range failed {
|
||||
require.NoError(t, setupErr[m], "mount%d", m)
|
||||
if len(f) > 0 {
|
||||
t.Logf("mount%d: %d calls failed, first: %s", m, len(f), f[0])
|
||||
}
|
||||
}
|
||||
both := 0
|
||||
for i := range names {
|
||||
if wrote[0][i] && wrote[1][i] {
|
||||
both++
|
||||
}
|
||||
}
|
||||
t.Logf("both mounts wrote %d of %d files", both, files)
|
||||
require.Positive(t, both, "the mounts never wrote the same file")
|
||||
|
||||
// The filer deletes the chunks an update dropped shortly after the update.
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
conn, err := grpc.NewClient(cluster.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
require.NoError(t, err)
|
||||
defer conn.Close()
|
||||
client := filer_pb.NewSeaweedFilerClient(conn)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
var broken []string
|
||||
for _, name := range names {
|
||||
if ctx.Err() != nil {
|
||||
broken = append(broken, "checks did not finish: "+ctx.Err().Error())
|
||||
break
|
||||
}
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: "/" + dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
broken = append(broken, fmt.Sprintf("%s: lookup: %v", name, err))
|
||||
continue
|
||||
}
|
||||
for _, chunk := range resp.Entry.GetChunks() {
|
||||
if !cluster.needleExists(ctx, t, chunk.GetFileIdString()) {
|
||||
broken = append(broken, fmt.Sprintf("%s: chunk %s is deleted", name, chunk.GetFileIdString()))
|
||||
}
|
||||
}
|
||||
if got, err := cluster.readFromFiler(ctx, "/"+dir+"/"+name); err != nil {
|
||||
broken = append(broken, fmt.Sprintf("%s: read: %v", name, err))
|
||||
} else if !bytes.Equal(got, content[name]) {
|
||||
broken = append(broken, fmt.Sprintf("%s: read %d bytes, want the %d written", name, len(got), len(content[name])))
|
||||
}
|
||||
}
|
||||
require.Empty(t, broken, "%d of %d staged files are broken", len(broken), files)
|
||||
}
|
||||
|
||||
// needleExists reports whether the volume server still has the needle; a
|
||||
// deleted one answers 404 (only readDeleted=true would serve it).
|
||||
func (c *dlmTestCluster) needleExists(ctx context.Context, t testing.TB, fid string) bool {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("http://127.0.0.1:%d/%s", c.volumePort, fid), nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
// readFromFiler reads a file whole through filer0's HTTP API.
|
||||
func (c *dlmTestCluster) readFromFiler(ctx context.Context, path string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPorts[0], path), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("status %s", resp.Status)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
@@ -29,6 +29,7 @@ type FuseTestFramework struct {
|
||||
mountProcess *managedProcess
|
||||
filerAddr string
|
||||
filerPort int
|
||||
masterPort int
|
||||
weedBinary string
|
||||
isSetup bool
|
||||
config *TestConfig
|
||||
@@ -74,6 +75,10 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
require.NoError(t, err)
|
||||
|
||||
filerPort := freePort(t)
|
||||
masterPort := freePort(t)
|
||||
for masterPort == filerPort {
|
||||
masterPort = freePort(t)
|
||||
}
|
||||
|
||||
return &FuseTestFramework{
|
||||
t: t,
|
||||
@@ -83,6 +88,7 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
logDir: filepath.Join(tempDir, "logs"),
|
||||
filerPort: filerPort,
|
||||
filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort),
|
||||
masterPort: masterPort,
|
||||
weedBinary: findWeedBinary(),
|
||||
isSetup: false,
|
||||
config: config,
|
||||
@@ -218,6 +224,11 @@ func (f *FuseTestFramework) GetFilerAddr() string {
|
||||
return f.filerAddr
|
||||
}
|
||||
|
||||
// GetMasterAddr returns the master HTTP address of the mini cluster.
|
||||
func (f *FuseTestFramework) GetMasterAddr() string {
|
||||
return fmt.Sprintf("127.0.0.1:%d", f.masterPort)
|
||||
}
|
||||
|
||||
// managedProcess is a started weed sub-command whose exit is watched, so a wait
|
||||
// for it to come up ends the moment it dies instead of burning its full timeout.
|
||||
type managedProcess struct {
|
||||
@@ -321,7 +332,7 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
args = append(args, "-v=4")
|
||||
}
|
||||
adminPort := freePort(f.t)
|
||||
for adminPort == f.filerPort {
|
||||
for adminPort == f.filerPort || adminPort == f.masterPort {
|
||||
adminPort = freePort(f.t)
|
||||
}
|
||||
args = append(args,
|
||||
@@ -330,6 +341,8 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-filer.port="+strconv.Itoa(f.filerPort),
|
||||
"-master.port="+strconv.Itoa(f.masterPort),
|
||||
"-master.port.grpc="+strconv.Itoa(f.masterPort+grpcPortOffset),
|
||||
"-s3=false",
|
||||
"-webdav=false",
|
||||
"-admin.ui=false",
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
//go:build linux
|
||||
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
mergePageSize = 8192
|
||||
mergePages = 32
|
||||
)
|
||||
|
||||
// mergeHTTP bounds every HTTP call the test makes to the cluster.
|
||||
var mergeHTTP = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// mergeVacuumHTTP outlasts a master vacuum, which waits up to 30 s per volume
|
||||
// for writes assigned to it to drain before compacting it.
|
||||
var mergeVacuumHTTP = &http.Client{Timeout: 3 * time.Minute}
|
||||
|
||||
// TestReadOpenHandleAfterChunkMerge covers a file a flush merged while a
|
||||
// descriptor kept it open. The merge rewrites the file into new chunks and the
|
||||
// filer deletes the chunks it replaced, so after a vacuum only the new chunks
|
||||
// exist. Reads through the open handle must find them at once, whether or not
|
||||
// the handle has a write it has not flushed yet.
|
||||
func TestReadOpenHandleAfterChunkMerge(t *testing.T) {
|
||||
config := DefaultTestConfig()
|
||||
// The client chunk cache could still hold a replaced chunk; the reads
|
||||
// below have to reach the volume server.
|
||||
config.CacheSizeMB = 0
|
||||
config.MountOptions = append(config.MountOptions, "-cacheCapacityMB=0")
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("CleanHandle", func(t *testing.T) {
|
||||
testReadOpenHandleAfterChunkMerge(t, framework, "merged_clean.dat", false)
|
||||
})
|
||||
t.Run("HandleWithUnflushedWrite", func(t *testing.T) {
|
||||
testReadOpenHandleAfterChunkMerge(t, framework, "merged_dirty.dat", true)
|
||||
})
|
||||
}
|
||||
|
||||
func testReadOpenHandleAfterChunkMerge(t *testing.T, framework *FuseTestFramework, name string, unflushedWrite bool) {
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
held, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err)
|
||||
defer held.Close()
|
||||
|
||||
version := make([]int, mergePages)
|
||||
whole := make([]byte, 0, mergePages*mergePageSize)
|
||||
for p := 0; p < mergePages; p++ {
|
||||
whole = append(whole, mergePagePattern(p, 0)...)
|
||||
}
|
||||
_, err = held.WriteAt(whole, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, held.Sync())
|
||||
|
||||
// Overlapping 16-page windows: each leaves part of every older window
|
||||
// visible, so the stored chunks soon add up to more than twice the file
|
||||
// size and the next flush merges them.
|
||||
var replaced []mergeChunk
|
||||
for i, start := range []int{0, 3, 6, 9, 12, 15} {
|
||||
before := mergeEntryChunks(t, framework, name)
|
||||
window := make([]byte, 0, 16*mergePageSize)
|
||||
for p := start; p < start+16; p++ {
|
||||
window = append(window, mergePagePattern(p, i+1)...)
|
||||
version[p] = i + 1
|
||||
}
|
||||
_, err = held.WriteAt(window, int64(start*mergePageSize))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, held.Sync())
|
||||
if after := mergeEntryChunks(t, framework, name); len(after) == 1 && after[0].ModifiedTsNs == 0 {
|
||||
replaced = before
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, replaced, "the overlapping writes never made a flush merge the chunks")
|
||||
|
||||
// The filer deletes the replaced chunks after the merge commits; vacuum
|
||||
// once they are deleted so that only the merged chunk is left on disk.
|
||||
master := framework.GetMasterAddr()
|
||||
require.Eventually(t, func() bool {
|
||||
for _, c := range replaced {
|
||||
if mergeChunkStatus(master, c, false) != http.StatusNotFound {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 30*time.Second, 200*time.Millisecond, "the replaced chunks were never deleted")
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := mergeVacuumHTTP.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=0.0001", master))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
for _, c := range replaced {
|
||||
if mergeChunkStatus(master, c, true) != http.StatusNotFound {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 4*time.Minute, time.Second, "vacuum left the replaced chunks on disk")
|
||||
|
||||
if unflushedWrite {
|
||||
_, err = held.WriteAt(mergePagePattern(mergePages-1, 99), int64((mergePages-1)*mergePageSize))
|
||||
require.NoError(t, err)
|
||||
version[mergePages-1] = 99
|
||||
}
|
||||
|
||||
// A second descriptor shares the handle the first one keeps open. Drop its
|
||||
// pages from the kernel cache so every read goes to the mount.
|
||||
reader, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
require.NoError(t, unix.Fadvise(int(reader.Fd()), 0, 0, unix.FADV_DONTNEED))
|
||||
started := time.Now()
|
||||
buf := make([]byte, mergePageSize)
|
||||
for p := 0; p < mergePages; p++ {
|
||||
n, err := reader.ReadAt(buf, int64(p*mergePageSize))
|
||||
require.NoError(t, err, "read page %d through the open handle after the merge", p)
|
||||
require.Equal(t, mergePageSize, n, "page %d", p)
|
||||
require.Equal(t, mergePagePattern(p, version[p]), buf, "page %d", p)
|
||||
}
|
||||
assert.Less(t, time.Since(started), 5*time.Second, "reads through the open handle waited on chunks the merge replaced")
|
||||
}
|
||||
|
||||
func mergePagePattern(page, version int) []byte {
|
||||
b := make([]byte, mergePageSize)
|
||||
for i := 0; i < len(b); i += 8 {
|
||||
binary.LittleEndian.PutUint32(b[i:], uint32(page))
|
||||
binary.LittleEndian.PutUint32(b[i+4:], uint32(version))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type mergeChunk struct {
|
||||
FileId string `json:"file_id"`
|
||||
ModifiedTsNs int64 `json:"modified_ts_ns"`
|
||||
Fid *struct {
|
||||
VolumeId uint32 `json:"volume_id"`
|
||||
FileKey uint64 `json:"file_key"`
|
||||
Cookie uint32 `json:"cookie"`
|
||||
} `json:"fid"`
|
||||
}
|
||||
|
||||
func (c mergeChunk) fileId() string {
|
||||
if c.FileId != "" || c.Fid == nil {
|
||||
return c.FileId
|
||||
}
|
||||
return fmt.Sprintf("%d,%x%08x", c.Fid.VolumeId, c.Fid.FileKey, c.Fid.Cookie)
|
||||
}
|
||||
|
||||
// mergeEntryChunks returns the chunks the filer stores for the file.
|
||||
func mergeEntryChunks(t *testing.T, framework *FuseTestFramework, name string) []mergeChunk {
|
||||
t.Helper()
|
||||
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/%s?metadata=true", framework.GetFilerAddr(), url.PathEscape(name)))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "filer metadata of %s", name)
|
||||
var entry struct {
|
||||
Chunks []mergeChunk `json:"chunks"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&entry))
|
||||
return entry.Chunks
|
||||
}
|
||||
|
||||
// mergeChunkStatus HEADs a chunk on the volume server holding it, and returns
|
||||
// 0 when it cannot ask. With readDeleted, a deleted needle still answers until
|
||||
// a vacuum removes it.
|
||||
func mergeChunkStatus(master string, c mergeChunk, readDeleted bool) int {
|
||||
fid := c.fileId()
|
||||
volumeId := strings.SplitN(fid, ",", 2)[0]
|
||||
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/dir/lookup?volumeId=%s", master, volumeId))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var lookup struct {
|
||||
Locations []struct {
|
||||
Url string `json:"url"`
|
||||
} `json:"locations"`
|
||||
}
|
||||
err = json.NewDecoder(resp.Body).Decode(&lookup)
|
||||
resp.Body.Close()
|
||||
if err != nil || len(lookup.Locations) == 0 {
|
||||
return 0
|
||||
}
|
||||
target := fmt.Sprintf("http://%s/%s", lookup.Locations[0].Url, fid)
|
||||
if readDeleted {
|
||||
target += "?readDeleted=true"
|
||||
}
|
||||
head, err := mergeHTTP.Head(target)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
head.Body.Close()
|
||||
return head.StatusCode
|
||||
}
|
||||
@@ -57,3 +57,61 @@ class VectorInstallerTest(unittest.TestCase):
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / name).read_bytes(), contents)
|
||||
|
||||
|
||||
class EnterpriseInstallerTest(unittest.TestCase):
|
||||
shell = VectorInstallerTest.shell
|
||||
|
||||
def enterprise(self, code, **env):
|
||||
return self.shell('ENTERPRISE=true; REPO=seaweedfs/artifactory; EDITION=-enterprise; ' + code, **env)
|
||||
|
||||
def test_asset_names(self):
|
||||
result = self.enterprise('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'rust_asset_name linux arm64; worker_asset_name linux amd64; '
|
||||
'vector_asset_name linux arm64; LARGE_DISK=true; '
|
||||
'go_asset_name darwin arm64; rust_asset_name linux amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'weed-enterprise-linux_amd64.tar.gz', 'weed-enterprise-windows_amd64.zip',
|
||||
'weed-volume-enterprise_linux_arm64.tar.gz', 'weed-worker-enterprise_linux_amd64.tar.gz',
|
||||
'weed-vector-enterprise_linux_arm64.tar.gz', 'weed-enterprise-darwin_arm64_large_disk.tar.gz',
|
||||
'weed-volume-enterprise_large_disk_linux_amd64.tar.gz',
|
||||
])
|
||||
|
||||
def test_open_source_go_asset_names(self):
|
||||
result = self.shell('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'LARGE_DISK=true; go_asset_name windows amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'linux_amd64.tar.gz', 'windows_amd64.zip', 'windows_amd64_large_disk.zip',
|
||||
])
|
||||
|
||||
def test_windows_weed_keeps_exe(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
directory = Path(directory)
|
||||
archive = directory / 'weed.zip'
|
||||
with zipfile.ZipFile(archive, 'w') as output:
|
||||
output.writestr('weed.exe', b'weed fixture')
|
||||
dest = directory / 'bin'
|
||||
result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; '
|
||||
'INSTALL_DIR="$DEST"; install_component weed windows amd64',
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix())
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / 'weed.exe').read_bytes(), b'weed fixture')
|
||||
|
||||
def test_rust_binaries_linux_only(self):
|
||||
for system, arch, supported in [
|
||||
('linux', 'amd64', True), ('linux', 'arm64', True),
|
||||
('darwin', 'arm64', False), ('windows', 'amd64', False),
|
||||
]:
|
||||
with self.subTest(system=system, arch=arch):
|
||||
for check in ['vector_platform_supported', 'rust_volume_platform_supported']:
|
||||
result = self.enterprise(f'{check} "$SYSTEM" "$ARCH"', SYSTEM=system, ARCH=arch)
|
||||
self.assertEqual(result.returncode == 0, supported, check)
|
||||
|
||||
def test_latest_version_skips_other_releases(self):
|
||||
tags = ['vfs-0.2.5', 'kvcache-filer-native-v0.1.0-preview.1', '4.54-rc1', '4.53', '4.52']
|
||||
listing = ''.join(f' "tag_name": "{tag}",\\n' for tag in tags)
|
||||
result = self.enterprise(f"curl() {{ printf '{listing}'; }}; get_latest_version")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), '4.53')
|
||||
@@ -0,0 +1,116 @@
|
||||
//go:build !windows
|
||||
|
||||
package metadata_subscribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// A peer removal is broadcast only to the filers connected to the master that
|
||||
// saw the peer go. A peer that dies while no master is up (or across a leader
|
||||
// change) is never removed: the filer keeps retrying its address, the peer's
|
||||
// delivery watermark stays at zero, and every aggregated subscriber of that
|
||||
// filer is held at the settled horizon (2 x LogFlushInterval) for good.
|
||||
func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
weedBinary := findWeedBinary()
|
||||
require.NotEmpty(t, weedBinary, "weed binary not found")
|
||||
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
if t.Failed() {
|
||||
t.Logf("logs kept at %s", testDir)
|
||||
return
|
||||
}
|
||||
os.RemoveAll(testDir)
|
||||
})
|
||||
|
||||
ports, err := testutil.AllocateMiniPorts(3)
|
||||
require.NoError(t, err)
|
||||
masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
|
||||
master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
|
||||
filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
|
||||
|
||||
masterArgs := []string{"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(masterPort),
|
||||
"-mdir=" + mkdir(t, testDir, "master"),
|
||||
"-peers=none"}
|
||||
masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
||||
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
||||
startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
|
||||
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
|
||||
filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
|
||||
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
|
||||
filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
|
||||
|
||||
// filer1 follows filer2: an entry created on filer2 reaches filer1
|
||||
createPeerEntry(t, filer2Address, "from-filer2")
|
||||
require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second),
|
||||
"filer1 never followed filer2")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
events := make(chan *filer_pb.SubscribeMetadataResponse, 1000)
|
||||
sinceNs := time.Now().UnixNano()
|
||||
go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events)
|
||||
arrived := make(map[string]time.Time)
|
||||
waitArrival := func(name string, within time.Duration) bool {
|
||||
deadline := time.After(within)
|
||||
for {
|
||||
if _, ok := arrived[name]; ok {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case ev := <-events:
|
||||
if n := ev.GetEventNotification().GetNewEntry(); n != nil {
|
||||
arrived[n.Name] = time.Now()
|
||||
}
|
||||
case <-deadline:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
createPeerEntry(t, filer1Address, "baseline")
|
||||
require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber")
|
||||
|
||||
// filer2 dies while no master is up, so no removal is ever broadcast
|
||||
stopProcess(masterProcess)
|
||||
stopProcess(filer2)
|
||||
startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
||||
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
||||
require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second))
|
||||
diedAt := time.Now()
|
||||
|
||||
// Unpatched, every event from here on arrives 2 x LogFlushInterval late.
|
||||
// The dead peer may hold reads for its unreachable bound plus the removal
|
||||
// grace (2 x LogFlushInterval each); after that delivery must be prompt.
|
||||
for i := 0; ; i++ {
|
||||
name := fmt.Sprintf("probe-%d", i)
|
||||
createPeerEntry(t, filer1Address, name)
|
||||
if waitArrival(name, 10*time.Second) {
|
||||
t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second))
|
||||
return
|
||||
}
|
||||
if time.Since(diedAt) > 6*time.Minute {
|
||||
t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,7 +104,9 @@ for i in $(seq 1 60); do
|
||||
if mountpoint -q "${MOUNT_DIR}"; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
# Not kill -0: sudo briefly runs with a root real uid while starting, so
|
||||
# signalling it fails with EPERM even though it is alive.
|
||||
if ! ps -p "${mount_pid}" >/dev/null; then
|
||||
echo "weed mount exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
|
||||
@@ -2,13 +2,16 @@ package acl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -63,12 +66,22 @@ func TestBucketOwnershipControlsLifecycle(t *testing.T) {
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
|
||||
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
deleted, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
raw, ok := awsmiddleware.GetRawResponse(deleted.ResultMetadata).(*smithyhttp.Response)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, http.StatusNoContent, raw.StatusCode)
|
||||
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
_, err = client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "OwnershipControlsNotFoundError")
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerPreferred)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerPreferred, getObjectOwnership(t, client, bucketName))
|
||||
}
|
||||
|
||||
// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
.work
|
||||
@@ -0,0 +1,87 @@
|
||||
# Runs the listing tools and listing strategies of
|
||||
# https://github.com/varveio/s3-listing-study against a local SeaweedFS.
|
||||
#
|
||||
# make test build weed, fetch the study and its pinned tools, start
|
||||
# weed server on free ports, run, stop
|
||||
# make run ENDPOINT=... run against an already running endpoint
|
||||
#
|
||||
# Needs: go, python3, node/npm, cargo, docker (swath image), curl.
|
||||
#
|
||||
# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by
|
||||
# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set
|
||||
# holds a key with a backslash.
|
||||
|
||||
STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git
|
||||
STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917
|
||||
WORK_DIR ?= $(CURDIR)/.work
|
||||
STUDY_DIR ?= $(WORK_DIR)/s3-listing-study
|
||||
TOOLS_DIR ?= $(WORK_DIR)/tools
|
||||
VENV ?= $(WORK_DIR)/venv
|
||||
WEED_BINARY ?= $(WORK_DIR)/weed
|
||||
TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath
|
||||
ACCESS_KEY ?= listing_study_access
|
||||
SECRET_KEY ?= listing_study_secret
|
||||
JOBS ?= 4
|
||||
ONLY ?=
|
||||
|
||||
.PHONY: test run build-weed study venv tools start-server stop-server clean
|
||||
|
||||
test: build-weed tools
|
||||
@$(MAKE) --no-print-directory start-server; status=$$?; \
|
||||
if [ $$status -eq 0 ]; then \
|
||||
$(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \
|
||||
fi; \
|
||||
$(MAKE) --no-print-directory stop-server; exit $$status
|
||||
|
||||
build-weed:
|
||||
mkdir -p $(WORK_DIR)
|
||||
cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) .
|
||||
|
||||
study:
|
||||
@if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \
|
||||
git init -q $(STUDY_DIR) && \
|
||||
git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \
|
||||
git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \
|
||||
fi
|
||||
|
||||
venv:
|
||||
@test -x $(VENV)/bin/python || python3 -m venv $(VENV)
|
||||
$(VENV)/bin/pip install -q -r requirements.txt
|
||||
|
||||
tools: study venv
|
||||
$(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS)
|
||||
|
||||
start-server:
|
||||
@mkdir -p $(WORK_DIR)
|
||||
@set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \
|
||||
set -- $$ports; \
|
||||
$(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \
|
||||
-master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=50 -volume.preStopSeconds=1 \
|
||||
-master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \
|
||||
-s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \
|
||||
> $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \
|
||||
echo $$4 > $(WORK_DIR)/s3.port; \
|
||||
for i in $$(seq 1 90); do \
|
||||
if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \
|
||||
echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \
|
||||
kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \
|
||||
sleep 1; \
|
||||
done; tail -50 $(WORK_DIR)/weed.log; exit 1
|
||||
|
||||
stop-server:
|
||||
@if [ -f $(WORK_DIR)/weed.pid ]; then \
|
||||
pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \
|
||||
for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \
|
||||
rm -f $(WORK_DIR)/weed.pid; \
|
||||
fi
|
||||
|
||||
run: study
|
||||
$(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \
|
||||
--access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \
|
||||
--study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \
|
||||
--tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \
|
||||
$(if $(ONLY),--only '$(ONLY)')
|
||||
|
||||
clean: stop-server
|
||||
rm -rf .work
|
||||
@@ -0,0 +1,29 @@
|
||||
# Cases that fail for reasons outside SeaweedFS. A listed case that passes is
|
||||
# reported as XPASS and fails the run, so this list has to shrink with fixes.
|
||||
# Format: <case name> # reason
|
||||
|
||||
# aws-cli prints "None" for a --query over a page with no Contents (the canary
|
||||
# has no root-level keys); the capsule normalizer reads it as a malformed row.
|
||||
aws-cli:s3api-v2-remainder:canary:/ # capsule normalizer cannot parse aws-cli's "None"
|
||||
|
||||
# The edge bucket holds zero-byte directory markers (markers/dir/, markers/empty/).
|
||||
# These modes list them as directories, not objects: rclone's --files-only and
|
||||
# lsf --files-only drop them, s5cmd types any key ending in "/" as DIR and the
|
||||
# capsule normalizer drops DIR rows from its recursive modes. The reference
|
||||
# probes check that SeaweedFS returns both markers as Contents.
|
||||
rclone:recursive-fastlist:edge:/ # tool hides directory marker objects
|
||||
rclone:recursive-hierarchical:edge:/ # tool hides directory marker objects
|
||||
rclone:recursive-walk:edge:/ # tool hides directory marker objects
|
||||
rclone:listv1:edge:/ # tool hides directory marker objects
|
||||
rclone:lsf:edge:/ # tool hides directory marker objects
|
||||
s5cmd:recursive:edge:/ # tool hides directory marker objects
|
||||
s5cmd:listv1:edge:/ # tool hides directory marker objects
|
||||
s5cmd:allversions:edge:/ # tool hides directory marker objects
|
||||
|
||||
# Deleting the last key under a prefix leaves its filer directory behind, and an
|
||||
# empty directory is still listed as a CommonPrefix until the filer's empty-folder
|
||||
# cleanup removes it (2 minutes by default). This is deliberate: directories made
|
||||
# through mount or the filer stay visible to S3 clients (dirHoldsOnlyHiddenEntries
|
||||
# in weed/s3api/s3api_object_handlers_list.go). AWS derives CommonPrefixes from
|
||||
# keys only, so prefix-discovery tools briefly see the deleted prefix.
|
||||
reference:study-churn:deleted-prefixes # empty directories are listed until cleanup
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Print N free TCP ports whose +10000 gRPC companions are free too, all below
|
||||
the Linux ephemeral range (32768+) that outgoing connections draw from."""
|
||||
|
||||
import random
|
||||
import socket
|
||||
import sys
|
||||
|
||||
|
||||
def free(port: int) -> bool:
|
||||
with socket.socket() as s:
|
||||
try:
|
||||
s.bind(("127.0.0.1", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
picked: list[int] = []
|
||||
taken: list[int] = []
|
||||
while len(picked) < int(sys.argv[1]):
|
||||
port = random.randrange(10000, 22000)
|
||||
pair = (port, port + 10000)
|
||||
if all(abs(p - t) > 20 for p in pair for t in taken) and all(free(p) for p in pair):
|
||||
picked.append(port)
|
||||
taken += pair
|
||||
print(*picked)
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the listing tools pinned by the s3-listing-study checkout.
|
||||
|
||||
Every artifact locator and sha256 comes from the study's own
|
||||
tools/<tool>/build/image.json, so bumping STUDY_REV bumps the tools with it.
|
||||
Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ALL_TOOLS = (
|
||||
"aws-cli",
|
||||
"minio-mc",
|
||||
"ps3",
|
||||
"rclone",
|
||||
"s3-fast-list",
|
||||
"s3kor",
|
||||
"s3p",
|
||||
"s4cmd",
|
||||
"s5cmd",
|
||||
"s7cmd",
|
||||
"swath",
|
||||
)
|
||||
|
||||
MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104"
|
||||
"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z."""
|
||||
|
||||
SOURCE_PATCHES = {
|
||||
"s3-fast-list": (
|
||||
"s3-fast-list/src/data_map.rs",
|
||||
(
|
||||
(
|
||||
" match ctx.data_map_channel.try_recv() {",
|
||||
" let list_tasks_running = ctx.all_list_tasks_is_running();\n"
|
||||
" match ctx.data_map_channel.try_recv() {",
|
||||
),
|
||||
(
|
||||
"!ctx.all_list_tasks_is_running() && !has_more_in_queue",
|
||||
"!list_tasks_running && !has_more_in_queue",
|
||||
),
|
||||
),
|
||||
),
|
||||
}
|
||||
"""Source fixes applied before a build. s3-fast-list's data map reads the channel
|
||||
and then the list-task state, so a last page sent and completed between the two
|
||||
reads is dropped and the run exits 0 with a short or empty listing. Reading the
|
||||
state first means a finished task's pages are already in the channel."""
|
||||
|
||||
|
||||
def patch_source(tool: str, work: Path) -> None:
|
||||
rel, edits = SOURCE_PATCHES[tool]
|
||||
path = work / rel
|
||||
text = path.read_text()
|
||||
for old, new in edits:
|
||||
if text.count(old) != 1:
|
||||
raise SystemExit(f"{tool}: {rel} no longer matches its source patch: {old!r}")
|
||||
text = text.replace(old, new)
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def fetch(url: str, sha256: str, dest: Path) -> Path:
|
||||
if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256:
|
||||
return dest
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
|
||||
with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
digest = hashlib.sha256(tmp.read_bytes()).hexdigest()
|
||||
if digest != sha256:
|
||||
tmp.unlink()
|
||||
raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}")
|
||||
tmp.rename(dest)
|
||||
return dest
|
||||
|
||||
|
||||
def executable(path: Path) -> None:
|
||||
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
def wrapper(path: Path, body: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"#!/bin/sh\n{body}\n")
|
||||
executable(path)
|
||||
|
||||
|
||||
def tar_member(archive: Path, member: str, dest: Path) -> None:
|
||||
with tarfile.open(archive) as tar:
|
||||
source = tar.extractfile(member)
|
||||
if source is None:
|
||||
raise SystemExit(f"{archive}: no member {member}")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(source.read())
|
||||
executable(dest)
|
||||
|
||||
|
||||
def install(tool: str, study: Path, tools_dir: Path) -> None:
|
||||
image = json.loads((study / "tools" / tool / "build" / "image.json").read_text())
|
||||
artifact = image["tool_artifact"]
|
||||
url, sha = artifact["locator"], artifact["sha256"]
|
||||
dl = tools_dir / "downloads"
|
||||
bin_dir = tools_dir / "bin"
|
||||
bin_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = url.rsplit("/", 1)[-1]
|
||||
stamp = tools_dir / "installed" / tool
|
||||
installed = sha + ("+patched" if tool in SOURCE_PATCHES else "")
|
||||
if stamp.exists() and stamp.read_text() == installed:
|
||||
print(f"{tool} {image.get('tool_version', '')} already installed")
|
||||
return
|
||||
|
||||
if tool == "aws-cli":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
work = tools_dir / "aws-cli-src"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
zf.extractall(work)
|
||||
for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"):
|
||||
executable(work / rel)
|
||||
subprocess.run(
|
||||
[
|
||||
str(work / "aws" / "install"),
|
||||
"--install-dir",
|
||||
str(tools_dir / "aws-cli"),
|
||||
"--bin-dir",
|
||||
str(tools_dir / "aws-cli-bin"),
|
||||
"--update",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
shutil.rmtree(work)
|
||||
elif tool == "minio-mc":
|
||||
# dl.min.io answers 410 Gone for every archived mc release, so build the
|
||||
# same release tag from source instead.
|
||||
if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z":
|
||||
raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}")
|
||||
subprocess.run(
|
||||
["go", "install", f"github.com/minio/mc@{MC_COMMIT}"],
|
||||
env={**os.environ, "GOBIN": str(bin_dir)},
|
||||
check=True,
|
||||
)
|
||||
elif tool == "ps3":
|
||||
shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3")
|
||||
executable(bin_dir / "pS3")
|
||||
elif tool == "rclone":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
member = f"{name.removesuffix('.zip')}/rclone"
|
||||
(bin_dir / "rclone").write_bytes(zf.read(member))
|
||||
executable(bin_dir / "rclone")
|
||||
elif tool == "s3kor":
|
||||
tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor")
|
||||
elif tool == "s5cmd":
|
||||
tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd")
|
||||
elif tool == "s7cmd":
|
||||
tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd")
|
||||
elif tool == "s3p":
|
||||
work = tools_dir / "s3p"
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
for f in ("package.json", "package-lock.json"):
|
||||
shutil.copyfile(study / "tools/s3p/build" / f, work / f)
|
||||
subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True)
|
||||
wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"')
|
||||
elif tool == "s4cmd":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
venv = tools_dir / "s4cmd-venv"
|
||||
subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True)
|
||||
subprocess.run(
|
||||
[
|
||||
str(venv / "bin" / "python"),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--quiet",
|
||||
"--require-hashes",
|
||||
"--no-deps",
|
||||
"-r",
|
||||
str(study / "tools/s4cmd/build/requirements.txt"),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
script = tools_dir / "s4cmd" / "s4cmd.py"
|
||||
with tarfile.open(archive) as tar:
|
||||
source = tar.extractfile("s4cmd-2.1.0/s4cmd.py")
|
||||
assert source is not None
|
||||
script.parent.mkdir(parents=True, exist_ok=True)
|
||||
script.write_bytes(source.read())
|
||||
wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"')
|
||||
elif tool == "s3-fast-list":
|
||||
archive = fetch(url, sha, dl / name)
|
||||
work = tools_dir / "s3-fast-list-src"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True)
|
||||
shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock")
|
||||
patch_source(tool, work)
|
||||
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True)
|
||||
for exe in ("s3-fast-list", "ks-tool"):
|
||||
shutil.copyfile(work / "target/release" / exe, bin_dir / exe)
|
||||
executable(bin_dir / exe)
|
||||
elif tool == "swath":
|
||||
ref = f"{url}@sha256:{sha}"
|
||||
subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL)
|
||||
cid = subprocess.run(
|
||||
["docker", "create", ref], check=True, capture_output=True, text=True
|
||||
).stdout.strip()
|
||||
try:
|
||||
dest = tools_dir / "swath"
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
dest.mkdir(parents=True)
|
||||
subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True)
|
||||
subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True)
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
else:
|
||||
raise SystemExit(f"unknown tool {tool}")
|
||||
stamp.parent.mkdir(parents=True, exist_ok=True)
|
||||
stamp.write_text(installed)
|
||||
print(f"installed {tool} {image.get('tool_version', '')}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--study", required=True, type=Path)
|
||||
parser.add_argument("--tools-dir", required=True, type=Path)
|
||||
parser.add_argument("--tools", default=",".join(ALL_TOOLS))
|
||||
args = parser.parse_args()
|
||||
for tool in filter(None, args.tools.split(",")):
|
||||
install(tool, args.study.resolve(), args.tools_dir.resolve())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.umask(0o022)
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
# Versions the pinned s3-listing-study uv.lock resolves for the capsule normalizers.
|
||||
duckdb==1.5.5
|
||||
ijson==3.5.1
|
||||
pyyaml==6.0.3
|
||||
zstandard==0.25.0
|
||||
botocore==1.44.5
|
||||
@@ -0,0 +1,922 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS.
|
||||
|
||||
Tool cases: every mode of every capsule under the study's tools/ is compiled by
|
||||
the capsule's own command.py (benchmark.adapters.compile_command, with the
|
||||
custom-endpoint wiring the study's replay-endpoint pilot established), run
|
||||
against SeaweedFS with the pinned tool build, normalized by the capsule's own
|
||||
normalize.py, and the normalized listing is compared exactly against the keys
|
||||
that were written.
|
||||
|
||||
Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing
|
||||
the strategies the study classifies tools by (serial pagination, delimiter
|
||||
prefix discovery, start-after range splitting) plus the request semantics in
|
||||
docs/s3-reference.md, checked against the byte-ordered key set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
REGION = "us-east-1"
|
||||
S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
|
||||
|
||||
def canary_keys() -> dict[str, int]:
|
||||
"""benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes."""
|
||||
return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)}
|
||||
|
||||
|
||||
EDGE_KEYS = [
|
||||
# docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks.
|
||||
"index.html",
|
||||
# docs/s3-reference.md examples.
|
||||
"test_file(3).png",
|
||||
"photos/2006/February/sample.jpg",
|
||||
"photos/2006/January/sample.jpg",
|
||||
"photos/2007/sample.jpg",
|
||||
"notes/summer/july",
|
||||
"notes/summer/august",
|
||||
"notes/winter",
|
||||
# Byte order around the '/' (0x2f) delimiter.
|
||||
"order/a",
|
||||
"order/a b",
|
||||
"order/a!b",
|
||||
"order/a-b",
|
||||
"order/a.b",
|
||||
"order/a/b",
|
||||
"order/a/c/d",
|
||||
"order/a0",
|
||||
"order/aa",
|
||||
"order/a~",
|
||||
"order/A",
|
||||
"order/b",
|
||||
# Zero-byte directory markers.
|
||||
"markers/dir/",
|
||||
"markers/dir/child",
|
||||
"markers/empty/",
|
||||
# Characters that need XML or URL escaping.
|
||||
"chars/space key",
|
||||
"chars/plus+key",
|
||||
"chars/percent%20key",
|
||||
"chars/amp&key",
|
||||
"chars/lt<gt>key",
|
||||
'chars/quote"key',
|
||||
"chars/apos'key",
|
||||
"chars/hash#key",
|
||||
"chars/question?key",
|
||||
"chars/equals=key",
|
||||
"chars/semicolon;key",
|
||||
"chars/colon:key",
|
||||
"chars/at@key",
|
||||
"chars/comma,key",
|
||||
"chars/star*key",
|
||||
"chars/bang!key",
|
||||
"chars/dollar$key",
|
||||
"chars/caret^key",
|
||||
"chars/backslash\\key",
|
||||
"chars/brace{key}",
|
||||
"chars/pipe|key",
|
||||
"chars/backtick`key",
|
||||
"chars/tilde~key",
|
||||
# UTF-8 byte order, which differs from UTF-16 order above U+FFFF.
|
||||
"unicode/café-nfc",
|
||||
"unicode/café-nfd",
|
||||
"unicode/日本語.txt",
|
||||
"unicode/。-halfwidth",
|
||||
"unicode/\U0001f600-emoji",
|
||||
"unicode/z-ascii",
|
||||
"deep/a/b/c/d/e/f/g/h/i/j/k.txt",
|
||||
"long/" + "/".join(["y" * 200] * 4),
|
||||
]
|
||||
|
||||
|
||||
def edge_keys() -> dict[str, int]:
|
||||
return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)}
|
||||
|
||||
|
||||
def body_for(key: str, size: int) -> bytes:
|
||||
seed = hashlib.sha256(key.encode()).digest()
|
||||
return (seed * (size // len(seed) + 1))[:size]
|
||||
|
||||
|
||||
def utf8_sorted(keys) -> list[str]:
|
||||
return sorted(keys, key=lambda k: k.encode("utf-8"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- signed HTTP
|
||||
|
||||
|
||||
class S3:
|
||||
def __init__(self, endpoint: str, access: str, secret: str) -> None:
|
||||
from botocore.auth import S3SigV4Auth
|
||||
from botocore.credentials import Credentials
|
||||
|
||||
self.endpoint = endpoint.rstrip("/")
|
||||
self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION)
|
||||
|
||||
def request(
|
||||
self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b""
|
||||
) -> tuple[int, bytes]:
|
||||
from botocore.awsrequest import AWSRequest
|
||||
|
||||
query = "&".join(
|
||||
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
|
||||
for k, v in (params or [])
|
||||
)
|
||||
url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "")
|
||||
req = AWSRequest(method=method, url=url, data=body)
|
||||
req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
|
||||
self.auth.add_auth(req)
|
||||
prepared = req.prepare()
|
||||
http = urllib.request.Request(prepared.url, data=body or None, method=method)
|
||||
for k, v in prepared.headers.items():
|
||||
http.add_header(k, v)
|
||||
try:
|
||||
with urllib.request.urlopen(http, timeout=60) as resp:
|
||||
return resp.status, resp.read()
|
||||
except urllib.error.HTTPError as err:
|
||||
return err.code, err.read()
|
||||
|
||||
def put_object(self, bucket: str, key: str, body: bytes) -> None:
|
||||
# The first writes to a new bucket can get a retryable 500 while its
|
||||
# volumes grow; SDK clients retry those the same way.
|
||||
for attempt in range(10):
|
||||
status, out = self.request("PUT", f"/{bucket}/{key}", body=body)
|
||||
if status < 500:
|
||||
break
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
if status != 200:
|
||||
raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}")
|
||||
|
||||
def delete_object(self, bucket: str, key: str) -> None:
|
||||
status, out = self.request("DELETE", f"/{bucket}/{key}")
|
||||
if status != 204:
|
||||
raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}")
|
||||
|
||||
def create_bucket(self, bucket: str) -> None:
|
||||
status, out = self.request("PUT", f"/{bucket}")
|
||||
if status not in (200, 409):
|
||||
raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}")
|
||||
|
||||
def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element:
|
||||
status, out = self.request("GET", f"/{bucket}", params)
|
||||
if status != 200:
|
||||
raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}")
|
||||
return ET.fromstring(out)
|
||||
|
||||
|
||||
def text(el: ET.Element, name: str) -> str | None:
|
||||
child = el.find(S3NS + name)
|
||||
return None if child is None else (child.text or "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Page:
|
||||
entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order
|
||||
truncated: bool
|
||||
token: str | None
|
||||
next_marker: str | None
|
||||
key_count: str | None
|
||||
raw: ET.Element
|
||||
|
||||
|
||||
def check_byte_order(values: list[str], what: str) -> None:
|
||||
for a, b in zip(values, values[1:]):
|
||||
assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}"
|
||||
|
||||
|
||||
def parse_page(root: ET.Element, url_encoded: bool) -> Page:
|
||||
dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s)
|
||||
entries: list[tuple[str, str]] = []
|
||||
for child in root:
|
||||
if child.tag == S3NS + "Contents":
|
||||
entries.append(("key", dec(text(child, "Key") or "")))
|
||||
elif child.tag == S3NS + "CommonPrefixes":
|
||||
entries.append(("prefix", dec(text(child, "Prefix") or "")))
|
||||
nm = text(root, "NextMarker")
|
||||
# XML carries Contents and CommonPrefixes as two lists, each in byte order;
|
||||
# their merged byte order is what pagination has to respect.
|
||||
for kind in ("key", "prefix"):
|
||||
check_byte_order([v for k, v in entries if k == kind], f"{kind} list")
|
||||
entries.sort(key=lambda e: e[1].encode())
|
||||
return Page(
|
||||
entries,
|
||||
text(root, "IsTruncated") == "true",
|
||||
text(root, "NextContinuationToken"),
|
||||
dec(nm) if nm is not None else None,
|
||||
text(root, "KeyCount"),
|
||||
root,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- expectations
|
||||
|
||||
|
||||
def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]:
|
||||
"""What a delimiter listing of prefix returns, in byte order."""
|
||||
out: dict[str, str] = {}
|
||||
for k in keys:
|
||||
if not k.startswith(prefix):
|
||||
continue
|
||||
rest = k[len(prefix) :]
|
||||
idx = rest.find(delimiter) if delimiter else -1
|
||||
if idx >= 0:
|
||||
out[prefix + rest[: idx + len(delimiter)]] = "prefix"
|
||||
else:
|
||||
out[k] = "key"
|
||||
return [(out[k], k) for k in utf8_sorted(out)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reference probes
|
||||
|
||||
|
||||
class Probe:
|
||||
def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None:
|
||||
self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys)
|
||||
|
||||
def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]:
|
||||
pages: list[Page] = []
|
||||
token = None
|
||||
while True:
|
||||
params = [("list-type", "2"), ("max-keys", str(max_keys)), *base]
|
||||
if url:
|
||||
params.append(("encoding-type", "url"))
|
||||
if token:
|
||||
params.append(("continuation-token", token))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), url)
|
||||
pages.append(page)
|
||||
n = len(page.entries)
|
||||
assert n <= max_keys, f"page holds {n} > max-keys {max_keys}"
|
||||
assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries"
|
||||
if not page.truncated:
|
||||
assert not page.token, "NextContinuationToken on an untruncated page"
|
||||
return pages
|
||||
assert page.token, "IsTruncated without NextContinuationToken"
|
||||
assert len(pages) < 10000, "pagination does not terminate"
|
||||
token = page.token
|
||||
|
||||
def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]:
|
||||
pages: list[Page] = []
|
||||
marker = None
|
||||
while True:
|
||||
params = [("max-keys", str(max_keys)), *base]
|
||||
if delimiter:
|
||||
params.append(("delimiter", delimiter))
|
||||
if marker is not None:
|
||||
params.append(("marker", marker))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
pages.append(page)
|
||||
assert len(page.entries) <= max_keys
|
||||
if not page.truncated:
|
||||
return pages
|
||||
if delimiter:
|
||||
assert page.next_marker, "truncated delimiter page without NextMarker"
|
||||
marker = page.next_marker
|
||||
else:
|
||||
marker = page.next_marker or page.entries[-1][1]
|
||||
assert len(pages) < 10000, "pagination does not terminate"
|
||||
|
||||
@staticmethod
|
||||
def flat(pages: list[Page]) -> list[tuple[str, str]]:
|
||||
return [e for p in pages for e in p.entries]
|
||||
|
||||
def expect_eq(self, got, want, what: str) -> None:
|
||||
if got != want:
|
||||
gs, ws = set(got), set(want)
|
||||
missing = [w for w in want if w not in gs][:5]
|
||||
extra = [g for g in got if g not in ws][:5]
|
||||
first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want)))
|
||||
raise AssertionError(
|
||||
f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; "
|
||||
f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}"
|
||||
)
|
||||
|
||||
# Serial pagination (aws-cli, mc, s3kor, s5cmd ls).
|
||||
def serial(self, prefix: str, max_keys: int) -> None:
|
||||
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got = self.flat(self.v2_all([("prefix", prefix)], max_keys))
|
||||
self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}")
|
||||
|
||||
# Delimiter view and page boundaries inside it.
|
||||
def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None:
|
||||
want = direct_view(self.keys, prefix, delimiter)
|
||||
if v1:
|
||||
got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter))
|
||||
else:
|
||||
got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys))
|
||||
self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}")
|
||||
|
||||
# Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes.
|
||||
def walk(self, max_keys: int, v1: bool = False) -> None:
|
||||
found: list[str] = []
|
||||
queue = [""]
|
||||
while queue:
|
||||
prefix = queue.pop()
|
||||
if v1:
|
||||
entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/"))
|
||||
else:
|
||||
entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys))
|
||||
for kind, value in entries:
|
||||
(queue.append if kind == "prefix" else found.append)(value)
|
||||
self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}")
|
||||
assert len(found) == len(set(found)), "walk returned a key twice"
|
||||
|
||||
# Range splitting (S3P, s3-fast-list, Swath): start-after cut points.
|
||||
def ranges(self, cuts: list[str], max_keys: int) -> None:
|
||||
cuts = sorted(set(cuts), key=lambda c: c.encode())
|
||||
bounds = [None, *cuts, None]
|
||||
found: list[str] = []
|
||||
for lo, hi in zip(bounds, bounds[1:]):
|
||||
token = None
|
||||
done = False
|
||||
while not done:
|
||||
params = [("list-type", "2"), ("max-keys", str(max_keys))]
|
||||
if token:
|
||||
params.append(("continuation-token", token))
|
||||
elif lo is not None:
|
||||
params.append(("start-after", lo))
|
||||
page = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
for _, key in page.entries:
|
||||
if lo is not None:
|
||||
assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}"
|
||||
if hi is not None and key.encode() > hi.encode():
|
||||
done = True
|
||||
break
|
||||
found.append(key)
|
||||
if not page.truncated:
|
||||
break
|
||||
token = page.token
|
||||
self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})")
|
||||
|
||||
def start_after_with_delimiter(self, prefix: str, start_after: str) -> None:
|
||||
want = [
|
||||
e
|
||||
for e in direct_view(self.keys, prefix, "/")
|
||||
if e[1].encode() > start_after.encode()
|
||||
]
|
||||
got = self.flat(
|
||||
self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000)
|
||||
)
|
||||
self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}")
|
||||
|
||||
def token_overrides_start_after(self) -> None:
|
||||
first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False)
|
||||
assert first.truncated and first.token
|
||||
params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)]
|
||||
plain = parse_page(self.s3.list_page(self.bucket, params), False)
|
||||
mixed = parse_page(
|
||||
self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]),
|
||||
False,
|
||||
)
|
||||
self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after")
|
||||
|
||||
def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None:
|
||||
"""ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys."""
|
||||
want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got: list[tuple[str, str]] = []
|
||||
key_marker = version_marker = None
|
||||
dec = urllib.parse.unquote_plus if url else (lambda v: v)
|
||||
for _ in range(10000):
|
||||
params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)]
|
||||
if delimiter:
|
||||
params.append(("delimiter", delimiter))
|
||||
if url:
|
||||
params.append(("encoding-type", "url"))
|
||||
if key_marker is not None:
|
||||
params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")]
|
||||
root = self.s3.list_page(self.bucket, params)
|
||||
page: list[tuple[str, str]] = []
|
||||
for child in root:
|
||||
if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"):
|
||||
page.append(("key", dec(text(child, "Key") or "")))
|
||||
elif child.tag == S3NS + "CommonPrefixes":
|
||||
page.append(("prefix", dec(text(child, "Prefix") or "")))
|
||||
assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}"
|
||||
for kind in ("key", "prefix"):
|
||||
check_byte_order([v for k, v in page if k == kind], f"version {kind} list")
|
||||
got += sorted(page, key=lambda e: e[1].encode())
|
||||
if text(root, "IsTruncated") != "true":
|
||||
break
|
||||
nk = text(root, "NextKeyMarker")
|
||||
assert nk, "truncated versions page without NextKeyMarker"
|
||||
key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker")
|
||||
self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}")
|
||||
|
||||
def url_encoding(self, prefix: str) -> None:
|
||||
want = [("key", k) for k in self.keys if k.startswith(prefix)]
|
||||
got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True))
|
||||
self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}")
|
||||
root = self.s3.list_page(
|
||||
self.bucket,
|
||||
[("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")],
|
||||
)
|
||||
for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")):
|
||||
raw = text(root, name)
|
||||
assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}"
|
||||
assert text(root, "EncodingType") == "url", "EncodingType not echoed"
|
||||
|
||||
|
||||
def deleted_prefixes(s3: S3, bucket: str) -> None:
|
||||
"""A prefix whose last key was deleted is no longer a CommonPrefix."""
|
||||
s3.create_bucket(bucket)
|
||||
keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"]
|
||||
for k in keys:
|
||||
s3.put_object(bucket, k, b"x")
|
||||
for k in ("gone/a/b", "gone/c", "half/y/z"):
|
||||
s3.delete_object(bucket, k)
|
||||
p = Probe(s3, bucket, ["keep", "half/x"])
|
||||
p.delimiter("", "/", 1000)
|
||||
p.delimiter("half/", "/", 1000)
|
||||
p.delimiter("gone/", "/", 1000)
|
||||
|
||||
|
||||
def run_probes(
|
||||
s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None
|
||||
) -> list[tuple[str, str | None]]:
|
||||
results: list[tuple[str, str | None]] = []
|
||||
|
||||
def check(name: str, fn, *a) -> None:
|
||||
if only and not only.search(name):
|
||||
return
|
||||
try:
|
||||
fn(*a)
|
||||
results.append((name, None))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
results.append((name, f"{type(exc).__name__}: {exc}"))
|
||||
|
||||
check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket)
|
||||
for bucket, keymap in buckets.items():
|
||||
p = Probe(s3, bucket, list(keymap))
|
||||
keys = p.keys
|
||||
tag = f"reference:{bucket}"
|
||||
for mk in (1, 7, 100, 1000):
|
||||
check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk)
|
||||
check(f"{tag}:url-encoding", p.url_encoding, "")
|
||||
check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after)
|
||||
for mk in (2, 1000):
|
||||
check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk)
|
||||
check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True)
|
||||
prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k})
|
||||
scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]]
|
||||
if bucket.endswith("edge"):
|
||||
scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"]
|
||||
for prefix in scopes:
|
||||
for mk in (1, 3, 1000):
|
||||
check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk)
|
||||
check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True)
|
||||
for prefix in ("", scopes[-1]):
|
||||
for delim in ("", "/"):
|
||||
for mk in (2, 1000):
|
||||
check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False)
|
||||
check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True)
|
||||
for d in ("-", "a", "ou"):
|
||||
check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000)
|
||||
check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5)
|
||||
# Cut points that are existing keys, absent keys between keys, prefixes,
|
||||
# and a common prefix with and without its trailing delimiter.
|
||||
step = max(1, len(keys) // 9)
|
||||
existing = keys[step::step]
|
||||
absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1]
|
||||
dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k]
|
||||
dirs += [d.rstrip("/") for d in dirs]
|
||||
for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)):
|
||||
for mk in (3, 1000):
|
||||
check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk)
|
||||
for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")):
|
||||
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa)
|
||||
if bucket.endswith("edge"):
|
||||
for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"):
|
||||
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa)
|
||||
check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/")
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tool capsules
|
||||
|
||||
|
||||
@dataclass
|
||||
class Case:
|
||||
tool: str
|
||||
mode: str
|
||||
bucket: str
|
||||
prefix: str
|
||||
status: str = ""
|
||||
detail: str = ""
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class ToolRunner:
|
||||
def __init__(self, args, study: Path, work: Path, endpoint: str) -> None:
|
||||
sys.path.insert(0, str(study / "benchmark" / "src"))
|
||||
from benchmark import adapters # noqa: PLC0415
|
||||
from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415
|
||||
|
||||
self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated
|
||||
self.study, self.work, self.endpoint = study, work, endpoint
|
||||
tools = Path(args.tools_dir).resolve()
|
||||
self.fixture_dir = work / "fixtures"
|
||||
self.path_map = {
|
||||
"/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"),
|
||||
"/usr/bin/mc": str(tools / "bin" / "mc"),
|
||||
"/usr/local/bin/pS3": str(tools / "bin" / "pS3"),
|
||||
"/usr/local/bin/rclone": str(tools / "bin" / "rclone"),
|
||||
"/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"),
|
||||
"/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"),
|
||||
"/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"),
|
||||
"/usr/local/bin/s3p": str(tools / "bin" / "s3p"),
|
||||
"/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"),
|
||||
"/s5cmd": str(tools / "bin" / "s5cmd"),
|
||||
"/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"),
|
||||
"/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"),
|
||||
"/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"),
|
||||
"/usr/bin/python3": sys.executable,
|
||||
"/opt/benchmark/tools/": str(study / "tools") + "/",
|
||||
}
|
||||
os.environ["PYTHONPATH"] = str(study / "benchmark" / "src")
|
||||
self.env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"AWS_ACCESS_KEY_ID": args.access_key,
|
||||
"AWS_SECRET_ACCESS_KEY": args.secret_key,
|
||||
"AWS_REGION": REGION,
|
||||
"AWS_DEFAULT_REGION": REGION,
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
"TZ": "UTC",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
"PYTHONPATH": str(study / "benchmark" / "src"),
|
||||
}
|
||||
|
||||
def localize(self, argv, bucket: str) -> list[str]:
|
||||
out = []
|
||||
path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"}
|
||||
for token in argv:
|
||||
for src, dst in path_map.items():
|
||||
if token == src or (src.endswith("/") and token.startswith(src)):
|
||||
token = dst + token[len(src) :]
|
||||
break
|
||||
out.append(token)
|
||||
return out
|
||||
|
||||
def adapter_dir(self, tool: str) -> Path:
|
||||
return self.study / "tools" / tool / "adapter"
|
||||
|
||||
def write_fixture_companions(self, bucket: str, keys: list[str]) -> None:
|
||||
"""The staged-fixture companions, built the way fixture_bundle.py builds them."""
|
||||
import duckdb # noqa: PLC0415
|
||||
|
||||
out = self.fixture_dir / bucket
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
out.mkdir(parents=True)
|
||||
shards = sorted({k.split("/", 1)[0] for k in keys})
|
||||
if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards):
|
||||
(out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards))
|
||||
parts = out / "parquet"
|
||||
parts.mkdir()
|
||||
with duckdb.connect() as con:
|
||||
con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)")
|
||||
con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys])
|
||||
con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)")
|
||||
sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter"))
|
||||
from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415
|
||||
|
||||
try:
|
||||
generate_hints(parts, out / "s3-fast-list-hints.input", segments=4)
|
||||
except FixtureHintsError as exc:
|
||||
print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True)
|
||||
|
||||
def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]:
|
||||
"""Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes
|
||||
the capsule refuses only because the replay server lacks the API."""
|
||||
ep = self.endpoint
|
||||
if tool == "rclone":
|
||||
return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env
|
||||
if tool == "minio-mc":
|
||||
return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION}
|
||||
if tool == "s3kor":
|
||||
return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env
|
||||
if tool == "s7cmd":
|
||||
return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env
|
||||
if tool == "s5cmd":
|
||||
return [argv[0], "--endpoint-url", ep, *argv[1:]], env
|
||||
return [*argv, "--endpoint-url", ep], env
|
||||
|
||||
def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]:
|
||||
config: dict[str, object] = {}
|
||||
for name, axis in adapter.modes[mode].axes.items():
|
||||
if isinstance(axis, self.stated):
|
||||
config[name] = 4
|
||||
if adapter.tool == "s5cmd" and mode == "fanout-with-dirs":
|
||||
initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)})
|
||||
if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials):
|
||||
raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set")
|
||||
config["shard_initials"] = "".join(initials)
|
||||
return config
|
||||
|
||||
def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]:
|
||||
signed = adapter.supports_signed
|
||||
inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool]
|
||||
config = self.config_for(adapter, case, mode, keys)
|
||||
argv, env = self.adapters.compile_command(
|
||||
self.adapter_dir(case.tool),
|
||||
case.tool,
|
||||
mode=mode,
|
||||
bucket=case.bucket,
|
||||
region=REGION,
|
||||
prefix=case.prefix,
|
||||
signed=signed,
|
||||
config=config,
|
||||
sink_dir=str(sink),
|
||||
artifact_path=artifact_path,
|
||||
visible_memory_gb=2.0,
|
||||
endpoint_url="" if inject else self.endpoint,
|
||||
)
|
||||
if inject:
|
||||
argv, env = self.inject_endpoint(case.tool, list(argv), env)
|
||||
case.detail = "endpoint injected; "
|
||||
argv = self.localize(argv, case.bucket)
|
||||
case.argv = argv
|
||||
sink.mkdir(parents=True, exist_ok=True)
|
||||
home = sink / "home"
|
||||
home.mkdir(exist_ok=True)
|
||||
manifest = adapter.modes[mode]
|
||||
stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout")
|
||||
with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err:
|
||||
proc = subprocess.run(
|
||||
argv,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
env={**self.env, **env, "HOME": str(home)},
|
||||
cwd=sink,
|
||||
timeout=300,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace")
|
||||
raise RuntimeError(f"exit {proc.returncode}: {tail}")
|
||||
product = sink / manifest.product_file if manifest.product_artifact else None
|
||||
return product, config
|
||||
|
||||
def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None:
|
||||
adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool)
|
||||
manifest = adapter.modes[case.mode]
|
||||
if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING:
|
||||
case.status = "skip"
|
||||
case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing"
|
||||
return
|
||||
if (case.tool, case.mode) in ROOT_ONLY and case.prefix:
|
||||
case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix"
|
||||
return
|
||||
fixture = self.fixture_dir / case.bucket
|
||||
needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode)
|
||||
if needs and not (fixture / needs).exists():
|
||||
case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set"
|
||||
return
|
||||
keys = list(expected)
|
||||
sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root"))
|
||||
shutil.rmtree(sink, ignore_errors=True)
|
||||
try:
|
||||
artifact_path = ""
|
||||
for step in adapter.requires.get(case.mode, ()):
|
||||
producer_sink = sink / f"requires-{step.mode}"
|
||||
self.execute(adapter, case, step.mode, producer_sink, keys)
|
||||
artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact])
|
||||
if manifest.inline:
|
||||
inline_sink = sink / f"inline-{manifest.inline}"
|
||||
self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path)
|
||||
inline_manifest = adapter.modes[manifest.inline]
|
||||
artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values())))
|
||||
product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path)
|
||||
except NotApplicable as exc:
|
||||
case.status, case.detail = "skip", str(exc)
|
||||
return
|
||||
except self.adapters.AdapterError as exc:
|
||||
if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc):
|
||||
case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}"
|
||||
return
|
||||
raise
|
||||
normalized = sink / "normalized.tsv"
|
||||
self.adapters.normalize_to_path(
|
||||
self.adapter_dir(case.tool),
|
||||
case.tool,
|
||||
case.mode,
|
||||
case.prefix,
|
||||
normalized,
|
||||
input_path=None if manifest.product_channel == "dataset" else product,
|
||||
dataset=product if manifest.product_channel == "dataset" else None,
|
||||
config=config,
|
||||
)
|
||||
compare(case, normalized.read_bytes(), expected, manifest.fields)
|
||||
|
||||
|
||||
INJECT_ENDPOINT = {
|
||||
"aws-cli": {"s3api-v1-text", "s3api-versions-text"},
|
||||
"minio-mc": {"versions-json", "find", "find-json"},
|
||||
"rclone": {"listv1"},
|
||||
"s3kor": {"list-versions"},
|
||||
"s4cmd": {"recursive", "shallow", "show-directory", "du"},
|
||||
"s5cmd": {"listv1", "allversions"},
|
||||
"s7cmd": {"all-versions"},
|
||||
}
|
||||
"""Capsule modes refused only because the study's replay server serves
|
||||
ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and
|
||||
HeadBucket too."""
|
||||
|
||||
NOT_A_LISTING = {
|
||||
("rclone", "debug"),
|
||||
("rclone", "walk-debug"),
|
||||
("s3p", "summarize"),
|
||||
("s4cmd", "du"),
|
||||
("s4cmd", "show-directory"),
|
||||
}
|
||||
"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize
|
||||
and du print one aggregate, and `ls -d` prints the directory itself."""
|
||||
|
||||
|
||||
class NotApplicable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
ROOT_ONLY = {
|
||||
("aws-cli", "s3api-v2-remainder"),
|
||||
("s5cmd", "fanout-fixture-with-dirs"),
|
||||
("s3-fast-list", "list-hinted-fixture"),
|
||||
}
|
||||
"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture."""
|
||||
|
||||
SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth")
|
||||
KEYS_ONLY = re.compile(r"rootkeys|remainder")
|
||||
"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes."""
|
||||
|
||||
|
||||
def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None:
|
||||
rows = []
|
||||
for line in normalized.split(b"\n"):
|
||||
if not line:
|
||||
continue
|
||||
f = line.split(b"\t")
|
||||
rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode()))
|
||||
scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)}
|
||||
keys = utf8_sorted(scope)
|
||||
shallow = bool(SHALLOW.search(case.mode))
|
||||
if shallow:
|
||||
view = direct_view(keys, case.prefix, "/")
|
||||
want_objects = [k for kind, k in view if kind == "key"]
|
||||
allowed_dirs = {k for kind, k in view if kind == "prefix"}
|
||||
else:
|
||||
want_objects = keys
|
||||
allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)}
|
||||
problems: list[str] = []
|
||||
objects, dirs = [], []
|
||||
for key, size, etag in rows:
|
||||
is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope
|
||||
(dirs if is_dir else objects).append((key, size, etag))
|
||||
got_keys = [k for k, _, _ in objects]
|
||||
dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set()
|
||||
if dup:
|
||||
problems.append(f"duplicate keys {sorted(dup)[:5]}")
|
||||
missing = [k for k in want_objects if k not in set(got_keys)]
|
||||
extra = [k for k in set(got_keys) if k not in set(want_objects)]
|
||||
if missing:
|
||||
problems.append(f"missing {len(missing)}: {missing[:5]}")
|
||||
if extra:
|
||||
problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}")
|
||||
bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs]
|
||||
if bad_dirs:
|
||||
problems.append(f"unexpected directory rows {bad_dirs[:5]}")
|
||||
if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs:
|
||||
problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}")
|
||||
for key, size, etag in objects:
|
||||
if key not in scope:
|
||||
continue
|
||||
want_size, want_etag = scope[key]
|
||||
if size != "-" and size != str(want_size):
|
||||
problems.append(f"{key!r} size {size} != {want_size}")
|
||||
if etag != "-" and etag != want_etag:
|
||||
problems.append(f"{key!r} etag {etag} != {want_etag}")
|
||||
case.status = "fail" if problems else "pass"
|
||||
case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
|
||||
|
||||
def load_expected_failures(path: Path | None) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
if path and path.exists():
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
name, _, reason = line.partition("#")
|
||||
out[name.strip()] = reason.strip()
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--endpoint", required=True)
|
||||
ap.add_argument("--access-key", required=True)
|
||||
ap.add_argument("--secret-key", required=True)
|
||||
ap.add_argument("--study", required=True)
|
||||
ap.add_argument("--tools-dir", required=True)
|
||||
ap.add_argument("--work-dir", required=True)
|
||||
ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath")
|
||||
ap.add_argument("--bucket-prefix", default="study")
|
||||
ap.add_argument("--expected-failures", type=Path)
|
||||
ap.add_argument("--skip-upload", action="store_true")
|
||||
ap.add_argument("--only", default="", help="regex over case names")
|
||||
ap.add_argument("--jobs", type=int, default=4)
|
||||
args = ap.parse_args()
|
||||
|
||||
study = Path(args.study).resolve()
|
||||
work = Path(args.work_dir).resolve()
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
s3 = S3(args.endpoint, args.access_key, args.secret_key)
|
||||
buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()}
|
||||
expected: dict[str, dict[str, tuple[int, str]]] = {}
|
||||
for bucket, keymap in buckets.items():
|
||||
expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()}
|
||||
if args.skip_upload:
|
||||
continue
|
||||
s3.create_bucket(bucket)
|
||||
start = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(16) as pool:
|
||||
list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap))
|
||||
print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True)
|
||||
|
||||
only = re.compile(args.only) if args.only else None
|
||||
results: list[tuple[str, str, str]] = []
|
||||
for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only):
|
||||
results.append((name, "fail" if err else "pass", err or ""))
|
||||
|
||||
runner = ToolRunner(args, study, work, args.endpoint)
|
||||
cases: list[Case] = []
|
||||
for bucket in buckets:
|
||||
runner.write_fixture_companions(bucket, list(buckets[bucket]))
|
||||
scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"]
|
||||
for tool in filter(None, args.tools.split(",")):
|
||||
adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool)
|
||||
for mode in adapter.modes:
|
||||
for prefix in scopes:
|
||||
case = Case(tool, mode, bucket, prefix)
|
||||
if only and not only.search(case_name(case)):
|
||||
continue
|
||||
cases.append(case)
|
||||
|
||||
def run(case: Case) -> Case:
|
||||
start = time.time()
|
||||
try:
|
||||
runner.run_case(case, expected[case.bucket])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
case.status = "fail"
|
||||
case.detail = f"{type(exc).__name__}: {exc}"[:1500]
|
||||
if os.environ.get("STUDY_DEBUG"):
|
||||
traceback.print_exc()
|
||||
print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True)
|
||||
return case
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
|
||||
for case in pool.map(run, cases):
|
||||
results.append((case_name(case), case.status, case.detail))
|
||||
|
||||
xfail = load_expected_failures(args.expected_failures)
|
||||
regressions, fixed = [], []
|
||||
counts: dict[str, int] = {}
|
||||
for name, status, detail in results:
|
||||
if status == "fail" and name in xfail:
|
||||
status = "xfail"
|
||||
elif status == "pass" and name in xfail:
|
||||
fixed.append(name)
|
||||
elif status == "fail":
|
||||
regressions.append((name, detail))
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
(work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False))
|
||||
print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
||||
for name in fixed:
|
||||
print(f"XPASS (remove from expected failures): {name}")
|
||||
for name, detail in regressions:
|
||||
print(f"FAIL {name}: {detail}")
|
||||
return 1 if regressions or fixed else 0
|
||||
|
||||
|
||||
def case_name(case: Case) -> str:
|
||||
return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "listing_study_admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "listing_study_access",
|
||||
"secretKey": "listing_study_secret"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": ["Read", "List"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -239,7 +239,7 @@ The tests are automatically run in GitHub Actions on every push/PR that affects
|
||||
**Workflow**: `.github/workflows/s3-parquet-tests.yml`
|
||||
|
||||
**Test Matrix**:
|
||||
- Python versions: 3.9, 3.11, 3.12
|
||||
- Python versions: 3.10, 3.11, 3.12
|
||||
- PyArrow integration tests (s3fs): 20 test combinations
|
||||
- PyArrow native S3 tests: 6 test scenarios ✅
|
||||
- Cross-filesystem compatibility tests: 12 test scenarios ✅ **NEW**
|
||||
|
||||
@@ -31,3 +31,7 @@ already-running server, and see the top of `run.sh` for the other overrides.
|
||||
to path-style access with a `sed` patch.
|
||||
- `NOT_ACCESSIBLE_BUCKET` is a real bucket carrying a deny-all bucket policy,
|
||||
which is how the suite's `AccessDenied` negative tests are satisfied.
|
||||
- CI runs Maven with `maven-settings.xml`, which adds Google's Maven Central
|
||||
mirror in front of Central, so a 429 from one falls through to the other.
|
||||
Locally, pass it with
|
||||
`MAVEN_ARGS="-s $PWD/test/s3/snowflake/maven-settings.xml"`.
|
||||
@@ -0,0 +1,28 @@
|
||||
<settings>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>central-fallback</id>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>google-maven-central</id>
|
||||
<url>https://maven-central.storage-download.googleapis.com/maven2/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>google-maven-central</id>
|
||||
<url>https://maven-central.storage-download.googleapis.com/maven2/</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
<activeProfiles>
|
||||
<activeProfile>central-fallback</activeProfile>
|
||||
</activeProfiles>
|
||||
</settings>
|
||||
@@ -432,3 +432,100 @@ func createVersionedTestBucket(t *testing.T, client *s3.Client) string {
|
||||
|
||||
return bucketName
|
||||
}
|
||||
|
||||
// TestTaggingReportsVersionId checks that tagging requests on a versioned
|
||||
// bucket answer with the version they acted on.
|
||||
func TestTaggingReportsVersionId(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := createVersionedTestBucket(t, client)
|
||||
defer cleanupTestBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "tagging-version-id"
|
||||
put := func(body string) string {
|
||||
resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(body),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return aws.ToString(resp.VersionId)
|
||||
}
|
||||
versionId1 := put("v1")
|
||||
versionId2 := put("v2")
|
||||
require.NotEmpty(t, versionId1)
|
||||
require.NotEmpty(t, versionId2)
|
||||
require.NotEqual(t, versionId1, versionId2)
|
||||
tagging := &types.Tagging{TagSet: []types.Tag{{Key: aws.String("k"), Value: aws.String("v")}}}
|
||||
|
||||
putResp, err := client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: aws.String(versionId1), Tagging: tagging,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionId1, aws.ToString(putResp.VersionId))
|
||||
|
||||
putResp, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), Tagging: tagging,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionId2, aws.ToString(putResp.VersionId))
|
||||
|
||||
getResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: aws.String(versionId1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionId1, aws.ToString(getResp.VersionId))
|
||||
|
||||
getResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionId2, aws.ToString(getResp.VersionId))
|
||||
|
||||
delResp, err := client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: aws.String(versionId1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, versionId1, aws.ToString(delResp.VersionId))
|
||||
}
|
||||
|
||||
// TestTaggingReportsNullVersionId checks that tagging an object written before
|
||||
// versioning was enabled reports the "null" version.
|
||||
func TestTaggingReportsNullVersionId(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := createTestBucket(t, client)
|
||||
defer cleanupTestBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "tagging-null-version-id"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader("before versioning"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
tagging := &types.Tagging{TagSet: []types.Tag{{Key: aws.String("k"), Value: aws.String("v")}}}
|
||||
|
||||
for _, versionId := range []*string{nil, aws.String("null")} {
|
||||
putResp, err := client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: versionId, Tagging: tagging,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "null", aws.ToString(putResp.VersionId))
|
||||
|
||||
getResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: versionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "null", aws.ToString(getResp.VersionId))
|
||||
|
||||
delResp, err := client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(objectKey), VersionId: versionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "null", aws.ToString(delResp.VersionId))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
s3-api-compat-tests/
|
||||
test-volume-data/
|
||||
compat-runner
|
||||
results.json
|
||||
weed-test.log
|
||||
weed-server.pid
|
||||
weed-server.started
|
||||
@@ -0,0 +1,132 @@
|
||||
# Runs https://github.com/tigrisdata-community/s3-api-compat-tests against a
|
||||
# local SeaweedFS S3 gateway and compares the report with expected_failures.txt.
|
||||
|
||||
.PHONY: all test help build-weed build-suite start-server stop-server run-suite test-with-server logs clean
|
||||
|
||||
SUITE_REPO := https://github.com/tigrisdata-community/s3-api-compat-tests.git
|
||||
SUITE_COMMIT := e846275e489ec30b324344d462e698252c3a1f67
|
||||
SUITE_DIR := s3-api-compat-tests
|
||||
SUITE_STAMP := $(SUITE_DIR)/.seaweedfs-prepared
|
||||
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT ?= 8335
|
||||
MASTER_PORT ?= 9335
|
||||
VOLUME_PORT ?= 8085
|
||||
FILER_PORT ?= 8889
|
||||
WEBDAV_PORT ?= 7335
|
||||
ADMIN_PORT ?= 23647
|
||||
SERVER_DIR := ./test-volume-data
|
||||
S3_CONFIG := ./test_s3.json
|
||||
# Succeeds while weed-server.pid names the server start-server launched: the
|
||||
# process at that pid must have the start time recorded with it, so a process
|
||||
# that reused the pid is left alone.
|
||||
OWN_SERVER_RUNNING = [ -f weed-server.pid ] && [ -s weed-server.started ] && \
|
||||
[ "$$(ps -p "$$(cat weed-server.pid)" -o lstart= 2>/dev/null)" = "$$(cat weed-server.started)" ]
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
all: test
|
||||
|
||||
test: test-with-server
|
||||
|
||||
help:
|
||||
@echo "s3-api-compat-tests (Tigris) against SeaweedFS"
|
||||
@echo ""
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " build-suite - Fetch the suite at $(SUITE_COMMIT) and build it"
|
||||
@echo " start-server - Start weed mini with S3 on port $(S3_PORT)"
|
||||
@echo " run-suite - Run the suite and check expected_failures.txt"
|
||||
@echo " stop-server - Stop the test server"
|
||||
@echo " test-with-server - Start server, run suite, stop server"
|
||||
@echo " logs - Show server logs"
|
||||
@echo " clean - Remove test artifacts"
|
||||
|
||||
build-weed:
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
|
||||
build-suite:
|
||||
@want="$(SUITE_COMMIT) $$(git hash-object seaweedfs-provider.patch)"; \
|
||||
if [ "$$(cat $(SUITE_STAMP) 2>/dev/null)" != "$$want" ]; then \
|
||||
rm -rf $(SUITE_DIR) && mkdir -p $(SUITE_DIR) && \
|
||||
git -C $(SUITE_DIR) init -q && \
|
||||
git -C $(SUITE_DIR) fetch -q --depth 1 $(SUITE_REPO) $(SUITE_COMMIT) && \
|
||||
git -C $(SUITE_DIR) checkout -q FETCH_HEAD && \
|
||||
git -C $(SUITE_DIR) apply ../seaweedfs-provider.patch && \
|
||||
echo "$$want" > $(SUITE_STAMP); \
|
||||
fi
|
||||
@cd $(SUITE_DIR) && go build -o ../compat-runner .
|
||||
|
||||
start-server: build-weed
|
||||
@if curl -sS --max-time 2 -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
|
||||
echo "port $(S3_PORT) is already served by another process; refusing to test against it"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if $(OWN_SERVER_RUNNING); then \
|
||||
echo "the server this Makefile started (pid $$(cat weed-server.pid)) is still running; run make stop-server first"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@rm -rf weed-server.pid weed-server.started $(SERVER_DIR)
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@$(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-ip=127.0.0.1 \
|
||||
-ip.bind=127.0.0.1 \
|
||||
-master.port=$(MASTER_PORT) \
|
||||
-volume.port=$(VOLUME_PORT) \
|
||||
-filer.port=$(FILER_PORT) \
|
||||
-webdav.port=$(WEBDAV_PORT) \
|
||||
-admin.port=$(ADMIN_PORT) \
|
||||
-admin.port.legacy=0 \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=$(S3_CONFIG) \
|
||||
-master.peers=none \
|
||||
> weed-test.log 2>&1 & echo $$! > weed-server.pid; \
|
||||
ps -p $$! -o lstart= > weed-server.started
|
||||
@for i in $$(seq 1 60); do \
|
||||
kill -0 $$(cat weed-server.pid) 2>/dev/null || { \
|
||||
echo "SeaweedFS server exited during startup"; \
|
||||
cat weed-test.log; \
|
||||
exit 1; \
|
||||
}; \
|
||||
if curl -sS --max-time 2 -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
|
||||
echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "SeaweedFS S3 server failed to start"; \
|
||||
cat weed-test.log; \
|
||||
$(MAKE) --no-print-directory stop-server; \
|
||||
exit 1
|
||||
|
||||
stop-server:
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
if $(OWN_SERVER_RUNNING); then \
|
||||
kill $$(cat weed-server.pid) 2>/dev/null || true; \
|
||||
for i in $$(seq 1 30); do $(OWN_SERVER_RUNNING) || break; sleep 1; done; \
|
||||
if $(OWN_SERVER_RUNNING); then \
|
||||
echo "the server this Makefile started (pid $$(cat weed-server.pid)) is still running after 30s; keeping weed-server.pid"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi; \
|
||||
rm -f weed-server.pid weed-server.started; \
|
||||
fi
|
||||
|
||||
run-suite: build-suite
|
||||
SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) \
|
||||
SEAWEEDFS_S3_ACCESS_KEY_ID=admin \
|
||||
SEAWEEDFS_S3_SECRET_ACCESS_KEY=admin \
|
||||
./compat-runner --providers=seaweedfs --output=results.json
|
||||
python3 check_results.py results.json expected_failures.txt
|
||||
|
||||
test-with-server:
|
||||
@$(MAKE) start-server || exit $$?; \
|
||||
$(MAKE) run-suite; TEST_RESULT=$$?; \
|
||||
$(MAKE) stop-server; STOP_RESULT=$$?; \
|
||||
if [ $$TEST_RESULT -ne 0 ]; then exit $$TEST_RESULT; fi; exit $$STOP_RESULT
|
||||
|
||||
logs:
|
||||
@test -f weed-test.log && tail -100 weed-test.log || echo "No log file found"
|
||||
|
||||
clean: stop-server
|
||||
@rm -rf $(SERVER_DIR) $(SUITE_DIR) compat-runner results.json weed-test.log weed-server.pid weed-server.started
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare an s3-api-compat-tests JSON report with expected_failures.txt.
|
||||
|
||||
Fails on any test that did not pass and is not listed with the status it
|
||||
ended with, and on any listed test that now passes or did not run, so the list
|
||||
only ever shrinks.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def load_expected(path):
|
||||
expected = {}
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
fields = line.split("#", 1)[0].split()
|
||||
if fields:
|
||||
expected[fields[0]] = fields[1] if len(fields) > 1 else "fail"
|
||||
return expected
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("usage: check_results.py <results.json> <expected_failures.txt>")
|
||||
with open(sys.argv[1]) as f:
|
||||
report = json.load(f)
|
||||
expected = load_expected(sys.argv[2])
|
||||
|
||||
providers = report.get("providers") or []
|
||||
results = providers[0].get("results") if providers else None
|
||||
if not results:
|
||||
sys.exit("no test results in the report: the suite could not start against the server")
|
||||
|
||||
seen = set()
|
||||
regressions, fixed = [], []
|
||||
for r in results:
|
||||
name, status = r["operation"], r["status"]
|
||||
seen.add(name)
|
||||
detail = r.get("error") or r.get("notes") or ""
|
||||
if status == "pass":
|
||||
if name in expected:
|
||||
fixed.append(name)
|
||||
elif name not in expected:
|
||||
regressions.append(f"{name} [{status}]: {detail}")
|
||||
elif status != expected[name]:
|
||||
regressions.append(f"{name} [{status}, listed as {expected[name]}]: {detail}")
|
||||
else:
|
||||
print(f"expected failure: {name} [{status}]")
|
||||
|
||||
missing = sorted(set(expected) - seen)
|
||||
print(f"{len(results)} tests, {len(regressions)} unexpected failures, "
|
||||
f"{len(fixed)} unexpected passes, {len(missing)} listed but not run")
|
||||
for line in regressions:
|
||||
print(f"UNEXPECTED FAILURE: {line}")
|
||||
for name in fixed:
|
||||
print(f"UNEXPECTED PASS: {name} (remove it from expected_failures.txt)")
|
||||
for name in missing:
|
||||
print(f"NOT RUN: {name} (remove it from expected_failures.txt)")
|
||||
if regressions or fixed or missing:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,18 @@
|
||||
# Tests from s3-api-compat-tests that SeaweedFS is expected to fail.
|
||||
# check_results.py fails CI on any other failure, and on any test listed here
|
||||
# that passes, so remove an entry together with the fix that makes it pass.
|
||||
# An entry expects the test to fail; "<name> error" expects its setup to fail
|
||||
# before the call it checks, and any other status is a regression.
|
||||
|
||||
# AWS answers GetBucketPolicyStatus on a bucket without a policy with
|
||||
# NoSuchBucketPolicy, and so does SeaweedFS; the suite expects a status.
|
||||
GetBucketPolicyStatus
|
||||
|
||||
# Glacier restore: SeaweedFS has no archive storage class to restore from, and
|
||||
# AWS also refuses RestoreObject on a STANDARD object (InvalidObjectState).
|
||||
RestoreObject
|
||||
|
||||
# SeaweedFS reads x-amz-rename-source as a literal key first and then, when it
|
||||
# starts with the request's bucket, as bucket/key, because AWS documents both
|
||||
# spellings. A leading slash is dropped since filer paths cannot hold it.
|
||||
RenameObject_SourceIsLiteralKey
|
||||
@@ -0,0 +1,19 @@
|
||||
diff --git a/providers.go b/providers.go
|
||||
index 149f94c..bc7aeeb 100644
|
||||
--- a/providers.go
|
||||
+++ b/providers.go
|
||||
@@ -180,6 +180,14 @@ func (p *Provider) CreateVersionedBucket(ctx context.Context, client *s3.Client,
|
||||
// If names is non-empty, only those named providers are returned.
|
||||
func LoadProviders(names []string) []Provider {
|
||||
all := []Provider{
|
||||
+ {
|
||||
+ Name: "seaweedfs",
|
||||
+ Endpoint: os.Getenv("SEAWEEDFS_S3_ENDPOINT"),
|
||||
+ Region: envOr("SEAWEEDFS_S3_REGION", "us-east-1"),
|
||||
+ PathStyle: true,
|
||||
+ AccessKey: os.Getenv("SEAWEEDFS_S3_ACCESS_KEY_ID"),
|
||||
+ SecretKey: os.Getenv("SEAWEEDFS_S3_SECRET_ACCESS_KEY"),
|
||||
+ },
|
||||
{
|
||||
Name: "tigris",
|
||||
Endpoint: "https://t3.storage.dev",
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "admin",
|
||||
"secretKey": "admin"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -44,12 +44,22 @@ func vacuumVolumes(t *testing.T) {
|
||||
require.NotEmpty(t, defaultConfig.MasterEndpoint, "vacuum needs a master endpoint; set MASTER_ENDPOINT")
|
||||
endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/vol/vacuum?garbageThreshold=0.001"
|
||||
httpClient := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := httpClient.Get(endpoint)
|
||||
require.NoError(t, err, "vacuum request to %s", endpoint)
|
||||
defer resp.Body.Close()
|
||||
_, err = io.Copy(io.Discard, resp.Body)
|
||||
require.NoError(t, err, "reading the vacuum response")
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "vacuum request to %s", endpoint)
|
||||
// The master runs one vacuum at a time and answers 409 to a request it
|
||||
// dropped because its automatic vacuum was running. Ask until ours runs.
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for {
|
||||
resp, err := httpClient.Get(endpoint)
|
||||
require.NoError(t, err, "vacuum request to %s", endpoint)
|
||||
_, err = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
require.NoError(t, err, "reading the vacuum response")
|
||||
if resp.StatusCode == http.StatusConflict && time.Now().Before(deadline) {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "vacuum request to %s", endpoint)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func requireVersionBody(t *testing.T, client *s3.Client, bucketName, objectKey, versionId string, want []byte, msg string) {
|
||||
|
||||
@@ -187,18 +187,34 @@ catalog.s3.path-style-access = "true"
|
||||
return configDir
|
||||
}
|
||||
|
||||
var icebergJars = []struct{ artifact, sha256 string }{
|
||||
{"iceberg-spark-runtime-3.5_2.12", "2aec4b78d806cc81d4e50325b351f6349c90de641544e644c6032f000938618b"},
|
||||
{"iceberg-aws-bundle", "f4398dcdfefc0b186b899d500651e6e5ac62abe26e25d5cbe28403eaaec24c20"},
|
||||
}
|
||||
|
||||
const (
|
||||
icebergVersion = "1.5.2"
|
||||
sparkJarsDir = "/opt/iceberg-jars"
|
||||
sparkJarsConfig = sparkJarsDir + "/iceberg-spark-runtime-3.5_2.12-" + icebergVersion + ".jar," +
|
||||
sparkJarsDir + "/iceberg-aws-bundle-" + icebergVersion + ".jar"
|
||||
)
|
||||
|
||||
func (env *TestEnvironment) startSparkContainer(t *testing.T, configDir string) {
|
||||
t.Helper()
|
||||
|
||||
mounts := testcontainers.Mounts(testcontainers.BindMount(configDir, "/config"))
|
||||
for _, jar := range icebergJars {
|
||||
path := testutil.MavenJar(t, "org.apache.iceberg", jar.artifact, icebergVersion, jar.sha256)
|
||||
mounts = append(mounts, testcontainers.BindMount(path, testcontainers.ContainerMountTarget(sparkJarsDir+"/"+filepath.Base(path))))
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
req := testcontainers.ContainerRequest{
|
||||
Image: "apache/spark:3.5.1",
|
||||
ExposedPorts: []string{"4040/tcp"},
|
||||
Mounts: testcontainers.Mounts(
|
||||
testcontainers.BindMount(configDir, "/config"),
|
||||
),
|
||||
Mounts: mounts,
|
||||
Env: map[string]string{
|
||||
"SPARK_LOCAL_IP": "localhost",
|
||||
},
|
||||
@@ -266,8 +282,6 @@ import sys
|
||||
spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
|
||||
python_path = os.path.join(spark_home, "python")
|
||||
py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
|
||||
ivy_dir = "/tmp/ivy"
|
||||
os.makedirs(ivy_dir, exist_ok=True)
|
||||
os.environ["AWS_REGION"] = "us-west-2"
|
||||
os.environ["AWS_DEFAULT_REGION"] = "us-west-2"
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = "test"
|
||||
@@ -281,8 +295,7 @@ from pyspark.sql import SparkSession
|
||||
|
||||
spark = (SparkSession.builder
|
||||
.appName("SeaweedFS Iceberg Test")
|
||||
.config("spark.jars.ivy", ivy_dir)
|
||||
.config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.2,org.apache.iceberg:iceberg-aws-bundle:1.5.2")
|
||||
.config("spark.jars", "%s")
|
||||
.config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
|
||||
.config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog")
|
||||
.config("spark.sql.catalog.iceberg.type", "rest")
|
||||
@@ -303,7 +316,7 @@ spark = (SparkSession.builder
|
||||
.getOrCreate())
|
||||
|
||||
%s
|
||||
`, icebergPort, s3Port, sql)
|
||||
`, sparkJarsConfig, icebergPort, s3Port, sql)
|
||||
|
||||
code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
|
||||
var output string
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var mavenCentralMirrors = []string{
|
||||
"https://repo1.maven.org/maven2",
|
||||
"https://maven-central.storage-download.googleapis.com/maven2",
|
||||
}
|
||||
|
||||
var (
|
||||
mavenDownloadBudget = 3 * time.Minute
|
||||
mavenAttemptTimeout = 75 * time.Second
|
||||
)
|
||||
|
||||
// MavenJar returns a local copy of a Maven Central jar, verified against its
|
||||
// pinned SHA-256. The jar is kept under the user cache directory so every test
|
||||
// in a run, and every run that restores that directory, shares one download.
|
||||
func MavenJar(t testing.TB, group, artifact, version, sha256Hex string) string {
|
||||
t.Helper()
|
||||
|
||||
cacheRoot, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
cacheRoot = os.TempDir()
|
||||
}
|
||||
dir := filepath.Join(cacheRoot, "seaweedfs-test-jars")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatalf("create the jar cache directory: %v", err)
|
||||
}
|
||||
name := fmt.Sprintf("%s-%s.jar", artifact, version)
|
||||
path := filepath.Join(dir, name)
|
||||
if fileSHA256(path) == sha256Hex {
|
||||
return path
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), mavenDownloadBudget)
|
||||
defer cancel()
|
||||
urlPath := fmt.Sprintf("%s/%s/%s/%s", strings.ReplaceAll(group, ".", "/"), artifact, version, name)
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3 && ctx.Err() == nil; attempt++ {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Duration(attempt) * 10 * time.Second):
|
||||
}
|
||||
}
|
||||
for _, mirror := range mavenCentralMirrors {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
attemptCtx, cancelAttempt := context.WithTimeout(ctx, mavenAttemptTimeout)
|
||||
lastErr = downloadVerified(attemptCtx, mirror+"/"+urlPath, path, sha256Hex)
|
||||
cancelAttempt()
|
||||
if lastErr == nil {
|
||||
return path
|
||||
}
|
||||
t.Logf("download %s: %v", name, lastErr)
|
||||
}
|
||||
}
|
||||
t.Fatalf("download %s from Maven Central: %v", name, lastErr)
|
||||
return ""
|
||||
}
|
||||
|
||||
func fileSHA256(path string) string {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return ""
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func downloadVerified(ctx context.Context, url, path, sha256Hex string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("GET %s: %s", url, resp.Status)
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
h := sha256.New()
|
||||
_, copyErr := io.Copy(io.MultiWriter(tmp, h), resp.Body)
|
||||
closeErr := tmp.Close()
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("GET %s: %w", url, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if got := hex.EncodeToString(h.Sum(nil)); got != sha256Hex {
|
||||
return fmt.Errorf("GET %s: sha256 %s, want %s", url, got, sha256Hex)
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDownloadVerified(t *testing.T) {
|
||||
body := []byte("jar contents")
|
||||
sum := sha256.Sum256(body)
|
||||
want := hex.EncodeToString(sum[:])
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/stall.jar" {
|
||||
<-r.Context().Done()
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/a.jar" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "a.jar")
|
||||
ctx := context.Background()
|
||||
|
||||
stallCtx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := downloadVerified(stallCtx, srv.URL+"/stall.jar", path, want); err == nil {
|
||||
t.Fatal("a stalled download should fail at the deadline")
|
||||
}
|
||||
|
||||
if err := downloadVerified(ctx, srv.URL+"/missing.jar", path, want); err == nil {
|
||||
t.Fatal("a 404 should fail")
|
||||
}
|
||||
if err := downloadVerified(ctx, srv.URL+"/a.jar", path, "00"); err == nil {
|
||||
t.Fatal("a checksum mismatch should fail")
|
||||
}
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 0 {
|
||||
t.Fatalf("failed downloads left %d files behind", len(entries))
|
||||
}
|
||||
|
||||
if err := downloadVerified(ctx, srv.URL+"/a.jar", path, want); err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if got := fileSHA256(path); got != want {
|
||||
t.Fatalf("sha256 %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMavenJarFallsBackPastStalledMirror(t *testing.T) {
|
||||
body := []byte("jar contents")
|
||||
sum := sha256.Sum256(body)
|
||||
want := hex.EncodeToString(sum[:])
|
||||
stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer stalled.Close()
|
||||
healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/org/example/demo/1.0/demo-1.0.jar" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
defer healthy.Close()
|
||||
|
||||
cache := t.TempDir()
|
||||
t.Setenv("XDG_CACHE_HOME", cache)
|
||||
t.Setenv("HOME", cache)
|
||||
t.Setenv("LocalAppData", cache)
|
||||
oldMirrors, oldBudget, oldAttempt := mavenCentralMirrors, mavenDownloadBudget, mavenAttemptTimeout
|
||||
mavenCentralMirrors = []string{stalled.URL, healthy.URL}
|
||||
mavenDownloadBudget, mavenAttemptTimeout = 10*time.Second, 200*time.Millisecond
|
||||
defer func() {
|
||||
mavenCentralMirrors, mavenDownloadBudget, mavenAttemptTimeout = oldMirrors, oldBudget, oldAttempt
|
||||
}()
|
||||
|
||||
path := MavenJar(t, "org.example", "demo", "1.0", want)
|
||||
if got := fileSHA256(path); got != want {
|
||||
t.Fatalf("sha256 %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
@@ -983,11 +983,21 @@ func TestTusAbortedPatchKeepsStoredChunks(t *testing.T) {
|
||||
require.Equal(t, http.StatusNoContent, patchResp.StatusCode)
|
||||
|
||||
// A vacuum reclaims whatever the filer deleted, so the file survives this
|
||||
// only if the chunks the session kept are still stored.
|
||||
vacuumResp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%s/vol/vacuum?garbageThreshold=0.001", testMasterPort))
|
||||
require.NoError(t, err)
|
||||
vacuumResp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, vacuumResp.StatusCode)
|
||||
// only if the chunks the session kept are still stored. The master answers
|
||||
// 409 to a request it dropped because its automatic vacuum was running, so
|
||||
// ask until ours runs.
|
||||
vacuumDeadline := time.Now().Add(time.Minute)
|
||||
for {
|
||||
vacuumResp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%s/vol/vacuum?garbageThreshold=0.001", testMasterPort))
|
||||
require.NoError(t, err)
|
||||
vacuumResp.Body.Close()
|
||||
if vacuumResp.StatusCode == http.StatusConflict && time.Now().Before(vacuumDeadline) {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
require.Equal(t, http.StatusOK, vacuumResp.StatusCode)
|
||||
break
|
||||
}
|
||||
|
||||
getResp, err := client.Get(cluster.FilerURL() + targetPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -28,7 +28,8 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
|
||||
const volumeID = uint32(83)
|
||||
const sourceNeedleID = uint64(333333)
|
||||
const sourceCookie = uint32(0xABCD0102)
|
||||
const clonedNeedleID = uint64(333334)
|
||||
const otherNeedleID = uint64(333334)
|
||||
const copyVolumeID = uint32(183)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
@@ -87,7 +88,24 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
|
||||
|
||||
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: clonedNeedleID,
|
||||
NeedleId: otherNeedleID,
|
||||
Size: size,
|
||||
NeedleBlob: blobResp.GetNeedleBlob(),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "blob header holds needle") {
|
||||
t.Fatalf("WriteNeedleBlob under another needle id should be refused, got %v", err)
|
||||
}
|
||||
if _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: otherNeedleID,
|
||||
}); err == nil {
|
||||
t.Fatalf("refused blob should not be indexed under needle %d", otherNeedleID)
|
||||
}
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, copyVolumeID, "")
|
||||
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
|
||||
VolumeId: copyVolumeID,
|
||||
NeedleId: sourceNeedleID,
|
||||
Size: size,
|
||||
NeedleBlob: blobResp.GetNeedleBlob(),
|
||||
})
|
||||
@@ -95,27 +113,27 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
|
||||
t.Fatalf("WriteNeedleBlob failed: %v", err)
|
||||
}
|
||||
|
||||
clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: clonedNeedleID,
|
||||
copiedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
|
||||
VolumeId: copyVolumeID,
|
||||
NeedleId: sourceNeedleID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err)
|
||||
t.Fatalf("VolumeNeedleStatus for copied needle failed: %v", err)
|
||||
}
|
||||
if clonedStatus.GetNeedleId() != sourceNeedleID {
|
||||
t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID)
|
||||
if copiedStatus.GetNeedleId() != sourceNeedleID {
|
||||
t.Fatalf("copied needle status id mismatch: got %d want %d", copiedStatus.GetNeedleId(), sourceNeedleID)
|
||||
}
|
||||
if clonedStatus.GetCookie() != sourceCookie {
|
||||
t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie)
|
||||
if copiedStatus.GetCookie() != sourceCookie {
|
||||
t.Fatalf("copied needle cookie mismatch: got %d want %d", copiedStatus.GetCookie(), sourceCookie)
|
||||
}
|
||||
|
||||
clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie))
|
||||
clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp)
|
||||
if clonedReadResp.StatusCode != 200 {
|
||||
t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode)
|
||||
copiedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(copyVolumeID, sourceNeedleID, sourceCookie))
|
||||
copiedReadBody := framework.ReadAllAndClose(t, copiedReadResp)
|
||||
if copiedReadResp.StatusCode != 200 {
|
||||
t.Fatalf("copied needle GET expected 200, got %d", copiedReadResp.StatusCode)
|
||||
}
|
||||
if string(clonedReadBody) != string(payload) {
|
||||
t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload))
|
||||
if string(copiedReadBody) != string(payload) {
|
||||
t.Fatalf("copied needle body mismatch: got %q want %q", string(copiedReadBody), string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -256,6 +256,17 @@ var pluginConfigSections = []pluginConfigSection{
|
||||
"collection_filter": stringValue,
|
||||
},
|
||||
},
|
||||
{
|
||||
// s3_lifecycle has no worker config form; its knobs are admin values
|
||||
jobType: "s3_lifecycle",
|
||||
prefix: "maintenance.s3_lifecycle",
|
||||
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": int64Value,
|
||||
"walker_interval_minutes": int64Value,
|
||||
"cluster_deletes_per_second": int64Value,
|
||||
"cluster_deletes_burst": int64Value,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func doubleValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
|
||||
|
||||
@@ -401,3 +401,56 @@ max_shard_transfers = 0
|
||||
t.Fatalf("max_shard_transfers = %d, want 0", got.GetInt64Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPluginConfigFromTomlS3Lifecycle(t *testing.T) {
|
||||
s := newPluginServer(t)
|
||||
seed := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "s3_lifecycle",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, RetryLimit: 1},
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
|
||||
},
|
||||
}
|
||||
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
v := tomlConfig(t, `
|
||||
[maintenance.s3_lifecycle]
|
||||
meta_log_retention_days = 0
|
||||
walker_interval_minutes = 30
|
||||
cluster_deletes_per_second = 50
|
||||
cluster_deletes_burst = 200
|
||||
retry_limit = 3
|
||||
`)
|
||||
if err := s.ApplyPluginConfigFromToml(v); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := s.GetPlugin().LoadJobTypeConfig("s3_lifecycle")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load: %v %v", cfg, err)
|
||||
}
|
||||
// an explicit 0 must replace the saved value, not read as unset
|
||||
if got, ok := cfg.AdminConfigValues["meta_log_retention_days"].GetKind().(*plugin_pb.ConfigValue_Int64Value); !ok || got.Int64Value != 0 {
|
||||
t.Errorf("meta_log_retention_days = %v, want int64 0", cfg.AdminConfigValues["meta_log_retention_days"])
|
||||
}
|
||||
if got := cfg.AdminConfigValues["walker_interval_minutes"].GetInt64Value(); got != 30 {
|
||||
t.Errorf("walker_interval_minutes = %d, want 30", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_per_second"].GetInt64Value(); got != 50 {
|
||||
t.Errorf("cluster_deletes_per_second = %d, want 50", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_burst"].GetInt64Value(); got != 200 {
|
||||
t.Errorf("cluster_deletes_burst = %d, want 200", got)
|
||||
}
|
||||
if cfg.AdminRuntime.RetryLimit != 3 {
|
||||
t.Errorf("retry_limit = %d, want 3", cfg.AdminRuntime.RetryLimit)
|
||||
}
|
||||
if !cfg.AdminRuntime.Enabled {
|
||||
t.Errorf("enabled flipped off by unrelated toml keys")
|
||||
}
|
||||
if cfg.UpdatedBy != "admin.toml" {
|
||||
t.Errorf("updated_by = %q, want admin.toml", cfg.UpdatedBy)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/notification/webhook"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// NotificationWebhook is a stored webhook target as shown to the browser:
|
||||
// the bearer token never leaves the server.
|
||||
type NotificationWebhook struct {
|
||||
webhook.ManagedTarget
|
||||
HasBearerToken bool `json:"has_bearer_token"`
|
||||
}
|
||||
|
||||
type NotificationWebhooksData struct {
|
||||
Webhooks []NotificationWebhook `json:"webhooks"`
|
||||
ConfigPath string `json:"config_path"`
|
||||
}
|
||||
|
||||
// NotificationWebhookRequest creates or replaces a target. An empty
|
||||
// bearer_token keeps the stored one unless clear_bearer_token is set.
|
||||
type NotificationWebhookRequest struct {
|
||||
webhook.ManagedTarget
|
||||
ClearBearerToken bool `json:"clear_bearer_token"`
|
||||
}
|
||||
|
||||
var notificationWebhooksMu sync.Mutex
|
||||
|
||||
func (s *AdminServer) readNotificationWebhooks(ctx context.Context) (*webhook.ManagedConfig, error) {
|
||||
var conf *webhook.ManagedConfig
|
||||
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
var buf bytes.Buffer
|
||||
err := filer.ReadEntry(s.masterClient, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, &buf)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
conf, err = webhook.ParseManagedConfig(buf.Bytes())
|
||||
return err
|
||||
})
|
||||
return conf, err
|
||||
}
|
||||
|
||||
func (s *AdminServer) saveNotificationWebhooks(ctx context.Context, conf *webhook.ManagedConfig) error {
|
||||
if err := conf.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
content, err := json.MarshalIndent(conf, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer.SaveInsideFiler(ctx, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, content)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetNotificationWebhooks(ctx context.Context) (*NotificationWebhooksData, error) {
|
||||
conf, err := s.readNotificationWebhooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := &NotificationWebhooksData{
|
||||
Webhooks: make([]NotificationWebhook, 0, len(conf.Targets)),
|
||||
ConfigPath: filer.NotificationWebhooksConfPath,
|
||||
}
|
||||
for _, t := range conf.Targets {
|
||||
hasToken := t.BearerToken != ""
|
||||
t.BearerToken = ""
|
||||
data.Webhooks = append(data.Webhooks, NotificationWebhook{ManagedTarget: t, HasBearerToken: hasToken})
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetNotificationWebhooksAPI(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := s.GetNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
s.putNotificationWebhook(w, r, "", true)
|
||||
}
|
||||
|
||||
func (s *AdminServer) UpdateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
s.putNotificationWebhook(w, r, mux.Vars(r)["name"], false)
|
||||
}
|
||||
|
||||
func (s *AdminServer) putNotificationWebhook(w http.ResponseWriter, r *http.Request, name string, create bool) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req NotificationWebhookRequest
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if create {
|
||||
name = req.Name
|
||||
} else if req.Name != "" && req.Name != name {
|
||||
writeJSONError(w, http.StatusBadRequest, "Webhook target name cannot be changed")
|
||||
return
|
||||
}
|
||||
target := req.ManagedTarget
|
||||
target.Name = name
|
||||
|
||||
notificationWebhooksMu.Lock()
|
||||
defer notificationWebhooksMu.Unlock()
|
||||
conf, err := s.readNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
index := -1
|
||||
for i := range conf.Targets {
|
||||
if conf.Targets[i].Name == name {
|
||||
index = i
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case create && index >= 0:
|
||||
writeJSONError(w, http.StatusConflict, "Webhook target already exists")
|
||||
return
|
||||
case !create && index < 0:
|
||||
writeJSONError(w, http.StatusNotFound, "Webhook target not found")
|
||||
return
|
||||
}
|
||||
if index >= 0 && target.BearerToken == "" && !req.ClearBearerToken {
|
||||
target.BearerToken = conf.Targets[index].BearerToken
|
||||
}
|
||||
if err := target.Validate(); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if index >= 0 {
|
||||
conf.Targets[index] = target
|
||||
} else {
|
||||
conf.Targets = append(conf.Targets, target)
|
||||
}
|
||||
if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
status := http.StatusOK
|
||||
if create {
|
||||
status = http.StatusCreated
|
||||
}
|
||||
hasToken := target.BearerToken != ""
|
||||
target.BearerToken = ""
|
||||
writeJSON(w, status, NotificationWebhook{ManagedTarget: target, HasBearerToken: hasToken})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
name := mux.Vars(r)["name"]
|
||||
|
||||
notificationWebhooksMu.Lock()
|
||||
defer notificationWebhooksMu.Unlock()
|
||||
conf, err := s.readNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
kept := conf.Targets[:0]
|
||||
for _, t := range conf.Targets {
|
||||
if t.Name != name {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
if len(kept) == len(conf.Targets) {
|
||||
writeJSONError(w, http.StatusNotFound, "Webhook target not found")
|
||||
return
|
||||
}
|
||||
conf.Targets = kept
|
||||
if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Webhook target deleted"})
|
||||
}
|
||||
@@ -126,6 +126,7 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
|
||||
|
||||
// File browser routes
|
||||
r.HandleFunc("/files", h.fileBrowserHandlers.ShowFileBrowser).Methods(http.MethodGet)
|
||||
r.HandleFunc("/notifications/webhooks", h.ShowNotificationWebhooks).Methods(http.MethodGet)
|
||||
|
||||
// Cluster management routes
|
||||
r.HandleFunc("/cluster/masters", h.clusterHandlers.ShowClusterMasters).Methods(http.MethodGet)
|
||||
@@ -275,6 +276,12 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet)
|
||||
filesApi.HandleFunc("/list-folders", h.fileBrowserHandlers.ListFolders).Methods(http.MethodGet)
|
||||
|
||||
webhooksApi := api.PathPrefix("/notifications/webhooks").Subrouter()
|
||||
webhooksApi.HandleFunc("", h.adminServer.GetNotificationWebhooksAPI).Methods(http.MethodGet)
|
||||
webhooksApi.Handle("", wrapWrite(h.adminServer.CreateNotificationWebhookAPI)).Methods(http.MethodPost)
|
||||
webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.UpdateNotificationWebhookAPI)).Methods(http.MethodPut)
|
||||
webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.DeleteNotificationWebhookAPI)).Methods(http.MethodDelete)
|
||||
|
||||
volumeApi := api.PathPrefix("/volumes").Subrouter()
|
||||
volumeApi.HandleFunc("/export", h.clusterHandlers.ExportClusterVolumes).Methods(http.MethodGet)
|
||||
volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost)
|
||||
@@ -386,6 +393,24 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// ShowNotificationWebhooks renders the filer webhook targets page
|
||||
func (h *AdminHandlers) ShowNotificationWebhooks(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
data, err := h.adminServer.GetNotificationWebhooks(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get webhook targets: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
component := app.NotificationWebhooks(*data)
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3VectorsBuckets renders the vector buckets page
|
||||
func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
@@ -62,6 +62,20 @@ func TestSetupRoutes_RegistersBucketLifecycleAPI_WithAuth(t *testing.T) {
|
||||
assertHasRoute(t, router, http.MethodDelete, "/api/s3/buckets/example/lifecycle")
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersNotificationWebhooks(t *testing.T) {
|
||||
for _, authRequired := range []bool{false, true} {
|
||||
router := mux.NewRouter()
|
||||
|
||||
newRouteTestAdminHandlers().SetupRoutes(router, authRequired, "admin", "password", "", "", true)
|
||||
|
||||
assertHasRoute(t, router, http.MethodGet, "/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodGet, "/api/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodPost, "/api/notifications/webhooks")
|
||||
assertHasRoute(t, router, http.MethodPut, "/api/notifications/webhooks/example")
|
||||
assertHasRoute(t, router, http.MethodDelete, "/api/notifications/webhooks/example")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersBucketPolicyAPI_NoAuth(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
|
||||
|
||||
Loaded 100 of 460 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user