Compare commits

..
Author SHA1 Message Date
Chris LuandClaude Opus 5.5 b89d7e36bf install.sh: offer the vector engine on linux only
Releases build weed-vector for linux amd64/arm64 only, so --component vector
on darwin or windows downloaded a missing asset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
2026-11-14 11:26:49 -08:00
485 changed files with 4158 additions and 41306 deletions

No files matched your search

+1 -44
View File
@@ -37,7 +37,7 @@ jobs:
with:
version: v3.18.4
- uses: actions/setup-go@v7
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
@@ -104,51 +104,8 @@ jobs:
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0
echo "S3 rollout settings render correctly"
echo "=== Testing S3 preStopSeconds ==="
if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
--set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then
echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1
fi
grep -q "must be at least 35" /tmp/s3-prestop-short.yaml
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
--set s3.enabled=true --set s3.preStopSeconds=20 \
--set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml
grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
--set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
--set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml
grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml
grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml
grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml
grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml
echo "S3 preStopSeconds renders correctly"
echo "=== Testing the S3 config checksum is stable across renders ==="
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
# Secret drew new keys for the checksum whenever lookup found nothing (on
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
- uses: actions/setup-python@v7
- uses: actions/setup-python@v6
with:
python-version: '3.13'
- uses: actions/cache/restore@v6
@@ -168,6 +168,9 @@ jobs:
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
- name: Run Rust unit tests (redb experimental cursor)
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
rust-unit-tests-windows:
name: Rust Unit Tests (Windows)
runs-on: windows-latest
@@ -205,6 +208,9 @@ jobs:
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
- name: Run Rust unit tests (redb experimental cursor)
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
rust-integration-tests:
name: Rust Integration Tests
runs-on: ubuntu-24.04
@@ -1,74 +0,0 @@
name: "S3 listing study tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/listing_study/**'
- '.github/workflows/s3-listing-study-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/listing_study/**'
- '.github/workflows/s3-listing-study-tests.yml'
concurrency:
group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
listing-study:
name: s3-listing-study tools and strategies
runs-on: ubuntu-24.04
timeout-minutes: 40
defaults:
run:
working-directory: test/s3/listing_study
steps:
- name: Check out code
uses: actions/checkout@v7
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Cache pinned listing tools
uses: actions/cache@v6
with:
path: test/s3/listing_study/.work/tools
key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }}
- name: Build weed and install the study's pinned tools
run: make build-weed tools
- name: Run the listing study against SeaweedFS
timeout-minutes: 15
run: make test
- name: Show logs on failure
if: failure()
run: |
tail -200 .work/weed.log || true
- name: Upload results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-listing-study-results
path: |
test/s3/listing_study/.work/run/results.json
test/s3/listing_study/.work/weed.log
retention-days: 14
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ['3.10', '3.11', '3.12']
python-version: ['3.9', '3.11', '3.12']
steps:
- name: Checkout code
+1 -11
View File
@@ -68,15 +68,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
- name: Cache suite Maven dependencies
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: ~/.m2/repository
key: snowflake-s3compat-m2-${{ hashFiles('test/s3/snowflake/run.sh') }}
restore-keys: |
snowflake-s3compat-m2-
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
- name: Install SeaweedFS
run: |
@@ -86,8 +78,6 @@ jobs:
- name: Run Snowflake S3Compat API tests
timeout-minutes: 20
env:
MAVEN_ARGS: -ntp -s ${{ github.workspace }}/test/s3/snowflake/maven-settings.xml
run: |
# Starts weed server, creates the buckets/objects the suite needs,
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
-6
View File
@@ -684,12 +684,6 @@ jobs:
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
- name: Cache Iceberg jars
uses: actions/cache@v6
with:
path: ~/.cache/seaweedfs-test-jars
key: iceberg-jars-${{ hashFiles('test/s3tables/catalog_spark/setup_test.go') }}
- name: Run go mod tidy
run: go mod tidy
@@ -1,57 +0,0 @@
name: "S3 Tigris Compatibility Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/tigris_compat/**'
- '.github/workflows/s3-tigris-compat-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/tigris_compat/**'
- '.github/workflows/s3-tigris-compat-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
jobs:
s3-tigris-compat-tests:
name: s3-api-compat-tests (Tigris)
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run s3-api-compat-tests
working-directory: test/s3/tigris_compat
run: make test
- name: Show server log on failure
if: failure()
working-directory: test/s3/tigris_compat
run: tail -200 weed-test.log || true
- name: Upload report
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-tigris-compat-results
path: test/s3/tigris_compat/results.json
if-no-files-found: ignore
retention-days: 7
+4 -4
View File
@@ -50,7 +50,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -410,7 +410,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -581,7 +581,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -810,7 +810,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -6,16 +6,12 @@ on:
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/telemetry-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/telemetry-integration.yml'
concurrency:
-2
View File
@@ -67,8 +67,6 @@ Download the latest binary from the [releases](https://github.com/seaweedfs/seaw
curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
```
For SeaweedFS Enterprise binaries, pass `--enterprise` to the script: `... | bash -s -- --enterprise`.
Then start a ready-to-use S3 object store:
```bash
+52 -51
View File
@@ -5,7 +5,7 @@ go 1.26.6
require (
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/pubsub v1.51.1
cloud.google.com/go/storage v1.69.0
cloud.google.com/go/storage v1.67.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.55.8
github.com/beorn7/perks v1.0.1 // indirect
@@ -84,24 +84,24 @@ require (
github.com/xdg-go/scram v1.2.0
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/api/v3 v3.7.3
go.etcd.io/etcd/client/v3 v3.7.3
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
go.mongodb.org/mongo-driver v1.17.9
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.49.0
gocloud.dev/pubsub/natspubsub v0.49.0
gocloud.dev v0.47.0
gocloud.dev/pubsub/natspubsub v0.46.0
gocloud.dev/pubsub/rabbitpubsub v0.47.0
golang.org/x/crypto v0.58.0
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
golang.org/x/image v0.46.0
golang.org/x/net v0.62.0
golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.49.0
golang.org/x/text v0.43.0 // indirect
golang.org/x/tools v0.52.0 // indirect
golang.org/x/tools v0.51.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.310.0
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 // indirect
google.golang.org/grpc v1.86.0-dev.0.20261027044109-d619b5d02089
google.golang.org/protobuf v1.36.12
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -123,10 +123,10 @@ require (
github.com/apache/iceberg-go v0.7.0
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/aws/aws-sdk-go-v2 v1.48.0
github.com/aws/aws-sdk-go-v2/config v1.33.9
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
github.com/aws/aws-sdk-go-v2 v1.47.7
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/cognusion/imaging v1.0.4
github.com/fluent/fluent-logger-golang v1.10.1
@@ -145,7 +145,7 @@ require (
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.32.0
github.com/pkg/sftp v1.13.12
github.com/rabbitmq/amqp091-go v1.16.0
github.com/rabbitmq/amqp091-go v1.15.0
github.com/rclone/rclone v1.75.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.23.0
@@ -162,7 +162,6 @@ require (
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
go.etcd.io/etcd/client/pkg/v3 v3.7.3
go.uber.org/atomic v1.12.0
go.yaml.in/yaml/v3 v3.0.6
golang.org/x/sync v0.24.0
golang.org/x/tools/godoc v0.1.0-deprecated
google.golang.org/grpc/security/advancedtls v1.0.0
@@ -185,7 +184,7 @@ require (
github.com/anchore/go-lzo v0.1.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/thrift v0.24.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -248,7 +247,7 @@ require (
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/oklog/ulid/v2 v2.1.2 // indirect
github.com/oklog/ulid/v2 v2.1.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
@@ -277,35 +276,37 @@ require (
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.45.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.1 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.6 // indirect
golang.org/x/mod v0.42.0 // indirect
gonum.org/v1/gonum v0.17.0 // indirect
)
require (
cel.dev/expr v0.25.3 // indirect
cloud.google.com/go/auth v0.24.1 // indirect
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a // indirect
cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect
cloud.google.com/go/compute/metadata v0.10.0 // indirect
cloud.google.com/go/iam v1.14.0 // indirect
cloud.google.com/go/monitoring v1.31.0 // indirect
cloud.google.com/go/monitoring v1.30.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.24.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
@@ -319,22 +320,22 @@ require (
github.com/andybalholm/cascadia v1.3.4 // indirect
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6
github.com/aws/smithy-go v1.28.9
github.com/aws/smithy-go v1.28.8
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/buengese/sgzip v0.1.1 // indirect
@@ -369,14 +370,14 @@ require (
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/errors v0.22.9 // indirect
github.com/go-openapi/strfmt v0.27.2 // indirect
github.com/go-openapi/errors v0.22.8 // indirect
github.com/go-openapi/strfmt v0.27.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.3 // indirect
github.com/go-resty/resty/v2 v2.17.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-json v0.11.2 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
@@ -386,7 +387,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.4.0
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -418,8 +419,8 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nats.go v1.49.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -433,7 +434,7 @@ require (
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.33
github.com/pierrec/lz4/v4 v4.1.29
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
@@ -455,7 +456,7 @@ require (
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
github.com/tarantool/go-iproto v1.1.0 // indirect
@@ -480,9 +481,9 @@ require (
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/bbolt v1.5.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
go.opentelemetry.io/otel v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
@@ -492,7 +493,7 @@ require (
go.uber.org/zap v1.28.0 // indirect
golang.org/x/term v0.47.0
golang.org/x/time v0.16.0
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
+112 -110
View File
@@ -4,8 +4,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs=
cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a h1:jW1ZLFy9oa5ai8LOI40z6s+DL1cO/L1KSm9hVGnibn8=
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88=
cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk=
cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY=
@@ -18,14 +18,14 @@ cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210Hl
cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w=
cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI=
cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0=
cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY=
cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU=
cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY=
cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM=
cloud.google.com/go/pubsub v1.51.1 h1:R3G1wCOxBO7jRpL8x2pdZMv1GAJDF6ax/m2zPOtvTNE=
cloud.google.com/go/pubsub v1.51.1/go.mod h1:y2T0IKtW1iWwVvazYaRpqOAFO4gy2+O7dTDt9TWY/5U=
cloud.google.com/go/pubsub/v2 v2.7.0 h1:MFrBTZZa6PDWZzCi4NJRsHKMm2w0a4oAaYNqwjgbQTE=
cloud.google.com/go/pubsub/v2 v2.7.0/go.mod h1:JaFvWNVRk3Knoil/4M1ECeLOaI9D8drbmJWypQlK5aM=
cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM=
cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g=
cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8=
cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
@@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 h1:zoPQ7KXGgzqeUmNWl+b6IwVAqsb5lROEJhqAmqXQtm0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3/go.mod h1:ei/sDx/dwDIUtZE3PQP1u8+qrA+CRy4q7qW+A2MbgB4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4=
@@ -60,8 +60,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY=
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ=
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA=
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI=
@@ -75,12 +75,14 @@ github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjw
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
github.com/Files-com/files-sdk-go/v3 v3.3.194/go.mod h1:rl0WumSN9gSo775DgvQv+wMQ8rlb0ES/1hU5jkMtLXg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 h1:rmzbFS2CkHq0y2L5LkMVWm+QXjjNEXxjmhrJqga/QGk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0 h1:eSgSuKdcpY7RTN1oXSzC6S39rPBImn03ubLMsWK4RRg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
github.com/IBM/go-sdk-core/v5 v5.23.1 h1:fxLusCG+GlGY1SM7BYAAiiSFhzKUs5zqs0P8knDqeFU=
github.com/IBM/go-sdk-core/v5 v5.23.1/go.mod h1:yO+OQpByKDLTvpEcsFFexgzpeR8eRfCFWAYzxkAu4bk=
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
@@ -171,50 +173,50 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 h1:4EezYQsIY0xPbD6u7TbQLfnpicSMQgA8n9paomVV4Zo=
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5/go.mod h1:Yr4VqXnyoyjWnOWJck8BOidkJtleQ1c8ypgd2n0epxE=
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 h1:D+Zmdt3T3P770MrqTmZvalbj6ADO7xA7n+qnNakGvrI=
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4/go.mod h1:dUYH3ia3qEO5ogaWDLp2tjtb4u4Y9xRbae7tOr1ClIA=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 h1:fD02zeo2Kc4hlFUvGX3FfA+gGOwd4BBcYgPDHDEzx9Y=
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0/go.mod h1:/lB2ILjoT5/q0zSwtHnrJPjXBvjalnEIl0IgSEdvHec=
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 h1:FghdY9ebeZf9iIan9J0xfBEID8arYibNBeZ1XHvs4oA=
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0/go.mod h1:wu7uOJ/2nAdenjvhHikeF6j0al99kfPVG6ZDU4e76OY=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -507,12 +509,12 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-openapi/errors v0.22.9 h1:HI9+SyVYiRzyeBQGv0CbWbQa+u2S0nny7G7AhY+UMiM=
github.com/go-openapi/errors v0.22.9/go.mod h1:R73q66sXP2smzAGgFOI4nqtusRK2xN5pYCjsij+KX58=
github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ=
github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E=
github.com/go-openapi/testify/v2 v2.8.0 h1:19QDx5b57p8KjO2E4sgs/woL4Akp+FGLRlVm2xn0Rho=
github.com/go-openapi/testify/v2 v2.8.0/go.mod h1:4gUN8jC+RqOE2qjW/vDmfyTvl5BCht4qL7oca0odrak=
github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I=
github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w=
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -544,8 +546,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
@@ -561,8 +563,8 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFig
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
@@ -647,8 +649,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -913,10 +915,10 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.15 h1:StSf9TINInaZtr4oww2+kXmfwa9SkN//g/LwS19/UJ0=
github.com/nats-io/nats-server/v2 v2.11.15/go.mod h1:zwhv8Y0PE3KHyKgznJc/9Xoai638SaJd83zzJ5GJn74=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -927,8 +929,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec=
github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -977,8 +979,8 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7c
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014=
github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c=
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
@@ -1049,8 +1051,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw=
github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSwiwT7XLAVc=
github.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=
github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=
@@ -1153,8 +1155,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
github.com/spiffe/go-spiffe/v2 v2.8.2 h1:jUEsvCMD6fH25J8K/w3q/XnIx8W1lb8+YLaEEHIjHmc=
github.com/spiffe/go-spiffe/v2 v2.8.2/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -1349,36 +1351,36 @@ go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
go.etcd.io/etcd/api/v3 v3.7.3 h1:2tumepXe+wi7Wab64uUO71K7ISQyUWLGC1DJbbc59sY=
go.etcd.io/etcd/api/v3 v3.7.3/go.mod h1:HOR9Qn4gb/1/hVAmYprmlLpvTe6ZgfgNS9Z/+Ty65po=
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
go.etcd.io/etcd/client/pkg/v3 v3.7.3 h1:5zMwNu3ycrzCeA5NwOfQbxFRhqq974chq1DrYMua4QI=
go.etcd.io/etcd/client/pkg/v3 v3.7.3/go.mod h1:/OhzqYrF0U7d1NOk55VMyUQGbYD1WoIe4bJwxb47wOk=
go.etcd.io/etcd/client/v3 v3.7.3 h1:UrXcetgkuSEocaBlQlulQRJlXLEnUAHuBRbHh0lue1Q=
go.etcd.io/etcd/client/v3 v3.7.3/go.mod h1:DcPxpqcpV8WIaRi/2waPCx0/Uq8rQUweDJaDYZk1gEE=
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 h1:MCcYL7J6Vt/X0kjqbMZkekCmwsurbQRbL69vkiye2lk=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0/go.mod h1:3jnStNwSufK+f5ktjL4EPcwtig4rtd81NS70lqHuXl8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0 h1:LaLVGJtIzHcvHjVs2OftmKcc+M9mdOqS8vZ1fBD1Yes=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0/go.mod h1:vF6H5sNDuQBJWgSX00iZgbIqk2MZZoVMH0ar5t5bOpw=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
@@ -1398,8 +1400,8 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syod
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v1.11.1 h1:SCrPqH9NE5CmBrOqpjNCiT0X5mGwMyGERXdogfR7Yjw=
go.opentelemetry.io/proto/otlp v1.11.1/go.mod h1:wq8W4aSf6bOy8RZyESOeQFr9Pu7wydWX/TY4sgvrTPg=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -1425,10 +1427,10 @@ go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
gocloud.dev v0.49.0 h1:02y4IBHX12xiCBK3Nv80IzuCmZXLsOfgpKwMEKzR9O4=
gocloud.dev v0.49.0/go.mod h1:uN+hz2x8ieyW5RjeMetvmpV/nRqFOr7DF0CKVY0Owp8=
gocloud.dev/pubsub/natspubsub v0.49.0 h1:Jk1bywJ6LvqvBbxOQ0qs0q4aa+ORgg1FzjhTy+5O4cI=
gocloud.dev/pubsub/natspubsub v0.49.0/go.mod h1:bmtxptoOPBNAgvTW9tWzoZvw9vzunDoNk6l4cHGd6nc=
gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA=
gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0=
gocloud.dev/pubsub/natspubsub v0.46.0 h1:cTmJnWOmhb3dYRzX7PvNkq2m5mQdBLFFxZbl/21pfjk=
gocloud.dev/pubsub/natspubsub v0.46.0/go.mod h1:hbEEsNt6mGQoonGQi39VOjHfx2f13DOmWouh4hGsfrU=
gocloud.dev/pubsub/rabbitpubsub v0.47.0 h1:iDSspbLlDbPL2avESbDfMNBK4r3B6H04Ejx+/hqh9JE=
gocloud.dev/pubsub/rabbitpubsub v0.47.0/go.mod h1:2tSZnGnGLPw2ayW4pVaUmnA1LCjP7rekpXPkaBUZt0o=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -1449,8 +1451,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e h1:HgI6W1OolKK6tcV74is+rl9SMT0njsHlp0TbQiGXI/A=
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e/go.mod h1:DBoxwWhjaW8DJJw/YeM4zpeOKMC1FMXSblsYy+1mFSw=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1606,8 +1608,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1629,10 +1631,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg=
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI=
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8=
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w=
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 h1:CXq2XEi5aEyLjjPZIwqrmQH8ZY4ctUwK7uyAx3mjX/U=
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941/go.mod h1:3gK0ehnhP0V7/x3ypGoLyT3d0n6TSi/EcsphIjHrcz4=
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 h1:DS/w3N8rsfrwxwgGSj6WhMsPo7aNQGmDx74J81zd0NQ=
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941/go.mod h1:AuGAdi9KnXss7xbLxIKhCvk+Ro/SxpET/iS45DNYN+M=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+31 -80
View File
@@ -7,14 +7,12 @@
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
# curl -fsSL ... | bash -s -- --enterprise --component all
#
# Options:
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
# --version VER Release version tag (default: latest)
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
# --dir DIR Installation directory (default: /usr/local/bin)
# --enterprise Install SeaweedFS Enterprise binaries (from seaweedfs/artifactory)
# --help Show this help message
set -euo pipefail
@@ -26,10 +24,6 @@ LARGE_DISK=false
INSTALL_DIR="/usr/local/bin"
WORKER_INSTALLED=false
VECTOR_INSTALLED=false
VOLUME_INSTALLED=false
ENTERPRISE=false
# Enterprise assets carry this after the binary name, e.g. weed-vector-enterprise_linux_amd64.tar.gz.
EDITION=""
# Colors (if terminal supports them)
if [ -t 1 ]; then
@@ -59,7 +53,6 @@ while [ $# -gt 0 ]; do
--version) VERSION="$2"; shift 2 ;;
--large-disk) LARGE_DISK=true; shift ;;
--dir) INSTALL_DIR="$2"; shift 2 ;;
--enterprise) ENTERPRISE=true; REPO="seaweedfs/artifactory"; EDITION="-enterprise"; shift ;;
--help|-h) usage ;;
*) error "Unknown option: $1. Use --help for usage." ;;
esac
@@ -88,22 +81,15 @@ detect_platform() {
}
# Get latest release tag from GitHub API
# The Enterprise repo also releases other products (vfs-*, previews), which can
# be its latest release, so take the newest plain version tag like 4.53.
get_latest_version() {
local url="https://api.github.com/repos/${REPO}/releases/latest"
if [ "$ENTERPRISE" = true ]; then
url="https://api.github.com/repos/${REPO}/releases?per_page=30"
fi
local body
if command -v curl &>/dev/null; then
body="$(curl -fsSL "$url")"
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
elif command -v wget &>/dev/null; then
body="$(wget -qO- "$url")"
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
else
error "Neither curl nor wget found. Please install one."
fi
echo "$body" | grep '"tag_name"' | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' | grep -E '^[0-9]+(\.[0-9]+)+$' | head -1 || true
}
# Download a file
@@ -124,22 +110,17 @@ go_asset_name() {
if [ "$LARGE_DISK" = true ]; then
suffix="${suffix}_large_disk"
fi
if [ "$ENTERPRISE" = true ]; then
suffix="weed-enterprise-${suffix}"
fi
if [ "$os" = "windows" ]; then
echo "${suffix}.zip"
else
echo "${suffix}.tar.gz"
fi
echo "${suffix}.tar.gz"
}
# Build Rust volume server asset name
rust_asset_name() {
local os="$1" arch="$2"
local prefix="weed-volume${EDITION}"
local prefix="weed-volume"
if [ "$LARGE_DISK" = true ]; then
prefix="${prefix}_large_disk"
prefix="weed-volume_large_disk"
else
prefix="weed-volume"
fi
local suffix="${os}_${arch}"
if [ "$os" = "windows" ]; then
@@ -149,19 +130,6 @@ rust_asset_name() {
fi
}
# Enterprise publishes the Rust volume server for linux only.
rust_volume_platform_supported() {
case "$2" in
amd64|arm64) ;;
*) return 1 ;;
esac
case "$1" in
linux) return 0 ;;
darwin|windows) [ "$ENTERPRISE" != true ] ;;
*) return 1 ;;
esac
}
# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops
# the installer on anything else: a rate limit or a network blip must not read
# as "this release predates the binary" and quietly skip it.
@@ -186,23 +154,19 @@ asset_exists() {
# maintains tables through the namespace and never opens a volume file.
worker_asset_name() {
local os="$1" arch="$2"
echo "weed-worker${EDITION}_${os}_${arch}.tar.gz"
echo "weed-worker_${os}_${arch}.tar.gz"
}
# The vector engine behind the S3 Vectors API, built like the worker.
# The vector engine behind the S3 Vectors API, published for linux only like
# the worker.
vector_asset_name() {
local os="$1" arch="$2"
if [ "$os" = "windows" ]; then
echo "weed-vector${EDITION}_${os}_${arch}.zip"
else
echo "weed-vector${EDITION}_${os}_${arch}.tar.gz"
fi
echo "weed-vector_${os}_${arch}.tar.gz"
}
vector_platform_supported() {
case "$1/$2" in
linux/amd64|linux/arm64) return 0 ;;
darwin/amd64|darwin/arm64|windows/amd64) [ "$ENTERPRISE" != true ] ;;
*) return 1 ;;
esac
}
@@ -222,10 +186,7 @@ install_component() {
download "$download_url" "${tmpdir}/${asset_name}"
info "Extracting ${asset_name}..."
case "$asset_name" in
*.zip) unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" ;;
*) tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" ;;
esac
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
# The Go release action puts the binary inside a directory
local weed_bin
@@ -238,16 +199,20 @@ install_component() {
fi
chmod +x "$weed_bin"
local dest_name="weed"
if [ "$os" = "windows" ]; then
dest_name="weed.exe"
fi
install_binary "$weed_bin" "$dest_name"
ok "Installed weed to ${INSTALL_DIR}/${dest_name}"
install_binary "$weed_bin" "weed"
ok "Installed weed to ${INSTALL_DIR}/weed"
;;
volume-rust)
rust_volume_platform_supported "$os" "$arch" || error "The Rust volume server is not available for ${os}/${arch}"
# Check platform support for Rust volume server
case "$os" in
linux|darwin|windows) ;;
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
esac
case "$arch" in
amd64|arm64) ;;
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
esac
asset_name="$(rust_asset_name "$os" "$arch")"
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
@@ -279,7 +244,6 @@ install_component() {
dest_name="weed-volume.exe"
fi
install_binary "$rust_bin" "$dest_name"
VOLUME_INSTALLED=true
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
;;
@@ -316,26 +280,19 @@ install_component() {
;;
vector)
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}"
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}. Supported: linux/amd64, linux/arm64"
asset_name="$(vector_asset_name "$os" "$arch")"
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
download "$download_url" "${tmpdir}/${asset_name}"
info "Extracting ${asset_name}..."
local vector_name="weed-vector"
if [ "$os" = "windows" ]; then
command -v unzip >/dev/null || error "unzip is required for Windows vector archives"
unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
vector_name="weed-vector.exe"
else
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
fi
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
local vector_bin
vector_bin="$(find "$tmpdir" -name "$vector_name" -type f | head -1)"
[ -n "$vector_bin" ] || error "Could not find $vector_name binary in archive"
vector_bin="$(find "$tmpdir" -name weed-vector -type f | head -1)"
[ -n "$vector_bin" ] || error "Could not find weed-vector binary in archive"
chmod +x "$vector_bin"
install_binary "$vector_bin" "$vector_name"
install_binary "$vector_bin" weed-vector
VECTOR_INSTALLED=true
ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
;;
@@ -392,11 +349,7 @@ main() {
case "$COMPONENT" in
all)
install_component "weed" "$os" "$arch"
if rust_volume_platform_supported "$os" "$arch"; then
install_component "volume-rust" "$os" "$arch"
else
warn "Skipping the Rust volume server: no build for ${os}/${arch}"
fi
install_component "volume-rust" "$os" "$arch"
# The worker is published for linux amd64/arm64 only, and only by
# releases new enough to carry it; skip either case rather than fail
# an install that has already put two binaries in place.
@@ -425,7 +378,7 @@ main() {
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
info " weed: ${INSTALL_DIR}/weed"
fi
if [ "$VOLUME_INSTALLED" = true ]; then
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
info " weed-volume: ${INSTALL_DIR}/weed-volume"
fi
if [ "$WORKER_INSTALLED" = true ]; then
@@ -438,9 +391,7 @@ main() {
info "Quick start:"
info " weed master # Start master server"
info " weed volume -mserver=localhost:9333 # Start Go volume server"
if [ "$VOLUME_INSTALLED" = true ]; then
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
fi
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
if [ "$WORKER_INSTALLED" = true ]; then
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
fi
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.54"
appVersion: "4.53"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.54.0
version: 4.53.1
+1 -3
View File
@@ -569,8 +569,6 @@ s3:
With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
Outside all-in-one mode, the volume Ingress and the default volume route target the `client-volume` Service, which lists only Ready volume pods. The `volume` Service also publishes pods that are not Ready yet, so their DNS records can exist before readiness. An explicit `backendRefs` entry that names the `volume` Service can therefore reach a volume server before it is Ready. Name the `client-volume` Service instead, with `volume.port`, and copy its `metadata.name` from `helm template` because long release names are truncated.
## Network Policies
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
@@ -720,7 +718,7 @@ The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverri
`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `<fullname>-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets.
The default `vector.metricsPort: 0` keeps TCP probes. Set `vector.metricsPort: 9329` to serve `/metrics`, `/health`, and `/ready` from the engine; readiness then checks master registration. Images older than 4.54 do not accept the metrics flag, so leave metrics off when pinning `image.tag` or `vector.imageOverride` to one. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
To provision vector buckets during installation and upgrades, set:
@@ -1,13 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.allInOne.enabled }}
{{- $s3PreStopSeconds := 0 }}
{{- if .Values.allInOne.s3.enabled }}
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
{{- if kindIs "invalid" $s3PreStopSeconds }}
{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }}
{{- end }}
{{- $s3PreStopSeconds = $s3PreStopSeconds | int }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -72,7 +64,7 @@ spec:
{{- tpl .Values.allInOne.tolerations . | nindent 8 }}
{{- end }}
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
terminationGracePeriodSeconds: 60
enableServiceLinks: false
{{- if .Values.allInOne.priorityClassName }}
priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
@@ -268,9 +260,6 @@ spec:
{{- if $auditLogConfig }}
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
{{- end }}
{{- if gt $s3PreStopSeconds 0 }}
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
{{- end }}
{{- end }}
{{- if .Values.allInOne.sftp.enabled }}
-sftp \
@@ -1,9 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.filer.enabled }}
{{- $s3PreStopSeconds := 0 }}
{{- if .Values.filer.s3.enabled }}
{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }}
{{- end }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
@@ -73,7 +69,7 @@ spec:
{{- end }}
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
terminationGracePeriodSeconds: 60
{{- if .Values.filer.priorityClassName }}
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
{{- end }}
@@ -282,9 +278,6 @@ spec:
{{- if .Values.filer.s3.auditLogConfig }}
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
{{- end }}
{{- if gt $s3PreStopSeconds 0 }}
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
{{- end }}
{{- end }}
-master={{ include "seaweedfs.masterServerArg" . }} \
{{- range .Values.filer.extraArgs }}
@@ -1,10 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.s3.enabled }}
{{- $preStopSeconds := .Values.s3.preStopSeconds | int }}
{{- $minGracePeriod := add $preStopSeconds 15 }}
{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }}
{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -159,9 +154,6 @@ spec:
{{- if .Values.s3.auditLogConfig }}
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
{{- end }}
{{- if gt $preStopSeconds 0 }}
-preStopSeconds={{ $preStopSeconds }} \
{{- end }}
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \
{{- if .Values.s3.icebergPort }}
-port.iceberg={{ .Values.s3.icebergPort }} \
@@ -1,5 +1,8 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.vector.enabled }}
{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }}
{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -1,6 +1,6 @@
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
@@ -2,7 +2,7 @@
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
{{- if and $volumeEnabled .Values.volume.ingress.enabled }}
{{- /* Determine service name based on deployment mode */}}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
@@ -1,42 +0,0 @@
{{- /* Ready-only backend for the volume ingress and HTTPRoute; the volume Service publishes not-ready pods for DNS.
Not named volume-client: every volume-<key> name belongs to a group under .Values.volumes. */}}
{{- if and .Values.volume.enabled (not .Values.allInOne.enabled) (or .Values.volume.ingress.enabled .Values.volume.httpRoute.enabled) }}
{{- $name := include "seaweedfs.componentName" (list . "client-volume") }}
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
{{- range $vname, $volume := $volumes }}
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
{{- if and $volume.enabled (eq (include "seaweedfs.componentName" (list $ $volumeName)) $name) }}
{{- fail (printf "The volume Ingress/HTTPRoute Service %q has the same name as the %s Service after truncation. Shorten fullnameOverride or the release name." $name $volumeName) }}
{{- end }}
{{- end }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
namespace: {{ include "seaweedfs.namespace" . | quote }}
labels:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: volume
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
{{- if .Values.volume.annotations }}
annotations:
{{- toYaml .Values.volume.annotations | nindent 4 }}
{{- end }}
spec:
clusterIP: None
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
{{- /* No metrics port, so the volume ServiceMonitor does not scrape each pod twice */}}
ports:
- name: "swfs-volume"
port: {{ .Values.volume.port }}
targetPort: {{ .Values.volume.port }}
protocol: TCP
selector:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: volume
{{- end }}
@@ -23,9 +23,6 @@ metadata:
{{- end }}
spec:
clusterIP: None
{{- /* The master hands out ${POD_NAME}.<this service> as soon as a volume server heartbeats, before it is Ready.
Ingress and HTTPRoute use the client-volume Service instead, which publishes only Ready pods. */}}
publishNotReadyAddresses: true
internalTrafficPolicy: {{ $volume.internalTrafficPolicy | default "Cluster" }}
ports:
- name: "swfs-volume"
+8 -6
View File
@@ -136,11 +136,13 @@ func TestVectorNetworkPolicyOverrides(t *testing.T) {
}
}
func TestVectorMetricsDefaultImage(t *testing.T) {
output, _ := render(t, "vector.enabled=true", "vector.metricsPort=9329")
for _, want := range []string{"--metrics-port=9329", "path: /ready", "path: /health"} {
if !strings.Contains(output, want) {
t.Fatalf("default image missing %q", want)
}
func TestVectorMetricsImageCompatibility(t *testing.T) {
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm is not installed")
}
out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput()
if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") {
t.Fatalf("incompatible image accepted: %v %s", err, out)
}
render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test")
}
@@ -1,107 +0,0 @@
package tests
import (
"fmt"
"os/exec"
"strings"
"testing"
)
func findDocument(documents []map[string]interface{}, kind, name string) map[string]interface{} {
for _, document := range documents {
if document["kind"] == kind && document["metadata"].(map[string]interface{})["name"] == name {
return document
}
}
return nil
}
func TestVolumeServicePublishesNotReadyPods(t *testing.T) {
_, documents := render(t)
service := findDocument(documents, "Service", "vector-test-seaweedfs-volume")
if service == nil {
t.Fatal("no volume service")
}
if service["spec"].(map[string]interface{})["publishNotReadyAddresses"] != true {
t.Fatal("volume pod DNS names must resolve before the pods are Ready")
}
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
t.Fatal("client-volume service rendered without ingress or httpRoute")
}
}
func TestVolumeIngressUsesReadyOnlyService(t *testing.T) {
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volume.metricsPort=9327")
client := findDocument(documents, "Service", "vector-test-seaweedfs-client-volume")
if client == nil {
t.Fatal("no client-volume service")
}
spec := client["spec"].(map[string]interface{})
if _, found := spec["publishNotReadyAddresses"]; found {
t.Fatal("client-volume must publish only Ready pods")
}
for _, port := range spec["ports"].([]interface{}) {
if port.(map[string]interface{})["name"] == "metrics" {
t.Fatal("client-volume must not expose metrics, or the ServiceMonitor scrapes each pod twice")
}
}
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
if ingress == nil {
t.Fatal("no volume ingress")
}
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
t.Fatalf("volume ingress backend is %v", backend)
}
route := findDocument(documents, "HTTPRoute", "vector-test-seaweedfs-volume")
if route == nil {
t.Fatal("no volume httpRoute")
}
rule := route["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})
if backend := rule["backendRefs"].([]interface{})[0].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
t.Fatalf("volume httpRoute backend is %v", backend)
}
}
func TestVolumeRoutingServiceDoesNotCollideWithVolumeGroups(t *testing.T) {
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volumes.client.replicas=1")
seen := map[string]bool{}
for _, document := range documents {
key := fmt.Sprint(document["kind"], "/", document["metadata"].(map[string]interface{})["name"])
if seen[key] {
t.Fatalf("%s rendered twice", key)
}
seen[key] = true
}
if !seen["Service/vector-test-seaweedfs-volume-client"] || !seen["Service/vector-test-seaweedfs-client-volume"] {
t.Fatal("volumes.client group Service and the routing Service must both render")
}
}
func TestVolumeRoutingServiceRejectsTruncatedCollision(t *testing.T) {
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm is not installed")
}
fullname := strings.Repeat("a", 49) + "-client"
output, err := exec.Command("helm", "template", "test", "..", "--set", "fullnameOverride="+fullname, "--set", "volume.ingress.enabled=true").CombinedOutput()
if err == nil || !strings.Contains(string(output), "same name as the volume Service") {
t.Fatalf("colliding routing Service accepted: %v %s", err, output)
}
}
func TestVolumeIngressAllInOne(t *testing.T) {
_, documents := render(t, "allInOne.enabled=true", "volume.ingress.enabled=true")
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
t.Fatal("client-volume service rendered in all-in-one mode")
}
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
if ingress == nil {
t.Fatal("no volume ingress")
}
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-all-in-one" {
t.Fatalf("all-in-one volume ingress backend is %v", backend)
}
}
+2 -17
View File
@@ -1070,10 +1070,6 @@ filer:
httpsPort: 0
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
# Seconds the gateway keeps serving after SIGTERM while its health checks
# return 503 (see s3.preStopSeconds). Added to the filer pod's 60s
# termination grace period.
preStopSeconds: 0
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
# set to the name of an existing kubernetes Secret with the s3 json config file
@@ -1122,17 +1118,8 @@ s3:
# maxUnavailable: 0
# maxSurge: 25%
updateStrategy: {}
# Time the s3 pod is given to shut down, including any preStop hook.
# With preStopSeconds set, it must be at least preStopSeconds + 15 (the
# in-flight request drain), or the chart refuses to render. A lifecycle
# preStop hook runs before SIGTERM, so add its duration on top; the chart
# cannot tell how long an exec hook takes. preStopSeconds replaces a
# preStop sleep hook.
# Time the s3 pod is given to shut down, including any preStop hook
terminationGracePeriodSeconds: 10
# Seconds the gateway keeps serving after SIGTERM while its health checks
# return 503, so the pod leaves the Service endpoints before it stops
# listening. 0 stops at once.
preStopSeconds: 0
bindAddress: 0.0.0.0
port: 8333
# add additional https port
@@ -1331,8 +1318,7 @@ s3:
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
# the Service endpoints before it receives SIGTERM; keep it shorter than
# terminationGracePeriodSeconds. s3.preStopSeconds does the same from
# inside the gateway and also fails its health checks.
# terminationGracePeriodSeconds.
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
# Example:
# lifecycle:
@@ -2025,7 +2011,6 @@ allInOne:
# The s3-secret.yaml template only reads from .Values.s3.credentials.
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables)
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
# You may specify buckets to be created during the install process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
+1 -3
View File
@@ -45,8 +45,6 @@ Rule of thumb: start with 3 masters, then grow volume servers with data.
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
- Run **3 filers** for HA
- Filers are stateless relative to each other — state lives in the DB
- No quorum among filers: 2 already survive one failure; the third keeps a spare while one is down for upgrades
- Give clients every filer address (e.g. `-filer=f1:8888,f2:8888,f3:8888`) or a load balancer that forwards both the HTTP and gRPC ports (8888 and 18888)
**S3 Servers**
@@ -79,7 +77,7 @@ Keeps housekeeping off the hot path.
|----------------|----------------------|-----------------------------------|
| Master | 3 | Raft quorum, topology, volume map |
| Volume Server | N (scale out) | Stores data needles |
| Filer | 3 (2 minimum) | Metadata gateway, no quorum |
| Filer | 3 | Metadata gateway |
| Metadata DB | HA cluster | Persists filer metadata |
| S3 Server | N (by throughput) | S3 API frontend |
| Admin | 1 | Background job coordinator |
+1084 -1055
View File
File diff suppressed because it is too large. Load diff

Before

Width:  |  Height:  |  Size: 54 KiB

After

Width:  |  Height:  |  Size: 55 KiB

+4 -624
View File
@@ -2302,113 +2302,6 @@
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Writable Volumes per Disk",
"description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.",
"type": "timeseries",
"id": 262,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)",
"range": true,
"refId": "A",
"legendFormat": "{{collection}} disks with 2+"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))",
"range": true,
"refId": "B",
"legendFormat": "{{collection}} max on one disk"
}
],
"pluginVersion": "10.3.1"
}
]
},
@@ -3442,102 +3335,6 @@
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Vacuum Dropped Needles",
"description": "Live needles vacuum dropped because their data could not be read back intact (CRC mismatch, malformed or short record), counted since the volume server started. Any non-zero value means a lost replica copy; the volume server log has each dropped key.",
"type": "timeseries",
"id": 267,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 54
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (instance, collection) (SeaweedFS_volumeServer_vacuuming_dropped_needles{cluster=~\"$cluster\"})",
"range": true,
"refId": "A",
"legendFormat": "{{instance}} {{collection}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
@@ -6832,423 +6629,6 @@
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Subscribe Delivery Lag",
"description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope",
"type": "timeseries",
"id": 263,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 71
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "A",
"legendFormat": "p99 {{scope}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "B",
"legendFormat": "p50 {{scope}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Watermark Hold Duration",
"description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)",
"type": "timeseries",
"id": 264,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 71
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "A",
"legendFormat": "p99 {{scope}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "B",
"legendFormat": "p50 {{scope}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Delivery Claim / Peer Low-Watermark Age",
"description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled",
"type": "timeseries",
"id": 265,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 79
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})",
"range": true,
"refId": "A",
"legendFormat": "claim {{instance}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})",
"range": true,
"refId": "B",
"legendFormat": "peer low {{kind}} {{instance}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Subscribers Resumed from Disk",
"description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log",
"type": "timeseries",
"id": 266,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 79
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "ops",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "{{scope}} {{instance}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
@@ -7262,7 +6642,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 87
"y": 71
},
"fieldConfig": {
"defaults": {
@@ -7358,7 +6738,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 87
"y": 71
},
"fieldConfig": {
"defaults": {
@@ -7454,7 +6834,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 95
"y": 79
},
"fieldConfig": {
"defaults": {
@@ -7550,7 +6930,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 95
"y": 79
},
"fieldConfig": {
"defaults": {
-3
View File
@@ -7143,7 +7143,6 @@ dependencies = [
"aes",
"aes-gcm",
"anyhow",
"arrow-array",
"arrow-cast",
"arrow-schema",
"async-trait",
@@ -7159,7 +7158,6 @@ dependencies = [
"hmac 0.12.1",
"http",
"hyper-util",
"jsonb",
"jsonwebtoken",
"lance",
"lance-datafusion",
@@ -7238,7 +7236,6 @@ dependencies = [
"futures",
"lance",
"lance-arrow",
"lance-index",
"lance-linalg",
"moka",
"object_store",
@@ -139,7 +139,6 @@ mod tests {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -168,7 +167,6 @@ mod tests {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let filtered = ann(&dataset, &table, &query).await.unwrap();
let exact = flat(&dataset, &table, &query).await.unwrap();
@@ -206,7 +204,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
+7 -17
View File
@@ -4,7 +4,6 @@
//! compares the elements as typed JSON values: Lance's compares their text, so
//! 1 would match "1".
use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::builder::{BooleanBuilder, StringBuilder};
@@ -20,21 +19,13 @@ use lance::deps::datafusion::logical_expr::{
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
/// Set `filter` on the scanner with the JSONPath functions swapped and the
/// conjuncts on `ready` indexed keys answered by their columns. Returns the
/// conjunction of those.
pub(crate) fn apply(
scanner: &mut Scanner,
filter: &str,
ready: &HashMap<String, String>,
) -> anyhow::Result<Option<Expr>> {
/// Set `filter` on the scanner with the JSONPath functions swapped.
pub(crate) fn apply(scanner: &mut Scanner, filter: &str) -> anyhow::Result<()> {
scanner.filter(filter)?;
let Some(expr) = scanner.get_expr_filter()? else {
return Ok(None);
};
let (expr, indexed) = crate::indexed_filter::rewrite(expr, ready);
scanner.filter_expr(swap(expr)?);
Ok(indexed)
if let Some(expr) = scanner.get_expr_filter()? {
scanner.filter_expr(swap(expr)?);
}
Ok(())
}
fn swap(expr: Expr) -> Result<Expr> {
@@ -277,7 +268,7 @@ mod tests {
async fn keys(dataset: &Dataset, filter: &str, swapped: bool) -> anyhow::Result<Vec<String>> {
let mut scanner = dataset.scan();
if swapped {
apply(&mut scanner, filter, &HashMap::new())?;
apply(&mut scanner, filter)?;
} else {
scanner.filter(filter)?;
}
@@ -404,7 +395,6 @@ mod tests {
apply(
&mut scanner,
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#,
&HashMap::new(),
)
.unwrap();
let mut swapped = Vec::new();
-78
View File
@@ -104,7 +104,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
}
}
@@ -199,83 +198,6 @@ mod tests {
assert_eq!(filtered.len(), 10);
}
/// A filtered query through the index returns every eligible row when
/// fewer than k match, though the probed partitions cannot hold them all:
/// 400 of 4000 rows match, the index has 63 partitions and probes at most
/// 32 of them.
#[tokio::test]
async fn filtered_ann_returns_every_match_below_k() {
let dir = TempDir::new().unwrap();
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
let tags: Vec<&str> = (0..4000)
.map(|i| if i % 10 == 0 { "rare" } else { "common" })
.collect();
let mut dataset = testing::write_dataset(&dir, &rows, &tags).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
build(
&mut dataset,
&table.vector_column,
INDEX_NAME,
partitions_for(4000),
false,
)
.await
.unwrap();
assert_eq!(partitions_for(4000), 63);
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
let q = Query {
filter: Some("tag = 'rare'".to_string()),
..exact(vector, 500)
};
let flat = query::flat(&dataset, &table, &q).await.unwrap();
assert_eq!(flat.len(), 400);
let ann = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(ann.len(), 400, "the index path dropped eligible rows");
for (f, a) in flat.iter().zip(ann.iter()) {
assert!((f.distance - a.distance).abs() < 1e-5);
}
}
}
/// When no more than k rows match, every match comes back with its own
/// distance, nearest first — not the rows the probes missed with an
/// infinite distance. Ten fragments of 400 rows, 5 of them matching.
#[tokio::test]
async fn filtered_ann_scores_every_match_at_or_below_k() {
let dir = TempDir::new().unwrap();
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
let tags: Vec<&str> = (0..4000)
.map(|i| if i % 800 == 0 { "rare" } else { "common" })
.collect();
let mut dataset = testing::write_dataset(&dir, &rows[..400], &tags[..400]).await;
for chunk in 1..10 {
let r = chunk * 400..(chunk + 1) * 400;
testing::append_dataset(&mut dataset, &rows[r.clone()], &tags[r]).await;
}
let table = VectorTable::resolve(&dataset, None, None).unwrap();
build(&mut dataset, &table.vector_column, INDEX_NAME, 63, false)
.await
.unwrap();
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
for k in [5, 10] {
let q = Query {
filter: Some("tag = 'rare'".to_string()),
..exact(vector, k)
};
let flat = query::flat(&dataset, &table, &q).await.unwrap();
assert_eq!(flat.len(), 5);
let ann = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(ann.len(), 5);
for (f, a) in flat.iter().zip(ann.iter()) {
assert!(
(f.distance - a.distance).abs() < 1e-5,
"k={k}: {flat:?} vs {ann:?}"
);
}
}
}
}
#[tokio::test]
async fn default_probes_take_the_largest_index() {
use lance::index::DatasetIndexExt;
@@ -1,599 +0,0 @@
//! Filters on indexed metadata keys (see `weed_lance_worker::metadata_index`).
//!
//! The gateway renders a string `$eq` or `$in` on a key as an OR of
//! `json_extract(metadata, P) = '"v"'` (or `IN`) and
//! `json_array_contains(metadata, P, '"v"')` over the same values. In the
//! parsed filter, such a top-level conjunct on a ready key is replaced by
//! `array_has` / `array_has_any` on the key's column, which its LABEL_LIST
//! index answers. Everything else, including `$ne` and `$nin`, stays on the
//! JSON path.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use anyhow::Result;
use jsonb::OwnedJsonb;
use lance::Dataset;
use lance::deps::datafusion::common::{Column, ScalarValue};
use lance::deps::datafusion::functions_nested::expr_fn::{array_has, array_has_any};
use lance::deps::datafusion::logical_expr::utils::split_conjunction_owned;
use lance::deps::datafusion::logical_expr::{BinaryExpr, Expr, Operator};
use lance::deps::datafusion::prelude::lit;
use weed_lance_worker::metadata_index::{self, State};
const METADATA_COLUMN: &str = "metadata";
/// The column of each ready indexed key of `dataset`, by metadata key.
pub(crate) fn ready_columns(dataset: &Dataset) -> HashMap<String, String> {
let schema: arrow_schema::Schema = dataset.schema().into();
metadata_index::indexed_keys(&schema)
.into_iter()
.filter(|k| k.state == Some(State::Ready))
.map(|k| (k.key, k.column))
.collect()
}
/// `filter` with each top-level conjunct that is a string `$eq` or `$in` on
/// a key in `ready` answered by the key's column instead, and the
/// conjunction of those replacements.
pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, Option<Expr>) {
if ready.is_empty() {
return (filter, None);
}
let mut indexed: Option<Expr> = None;
let mut rewritten: Option<Expr> = None;
for conjunct in split_conjunction_owned(filter) {
let replacement = string_membership(&conjunct).and_then(|(key, values)| {
let column = Expr::Column(Column::from_name(ready.get(&key)?));
Some(match values.as_slice() {
[one] => array_has(column, lit(one.as_str())),
_ => {
let values: Vec<ScalarValue> = values
.iter()
.map(|v| ScalarValue::Utf8(Some(v.clone())))
.collect();
let list = ScalarValue::new_list(&values, &arrow_schema::DataType::Utf8, true);
array_has_any(column, Expr::Literal(ScalarValue::List(list), None))
}
})
});
let part = match replacement {
Some(expr) => {
indexed = Some(and(indexed, expr.clone()));
expr
}
None => conjunct,
};
rewritten = Some(and(rewritten, part));
}
(rewritten.unwrap_or(lit(true)), indexed)
}
fn and(left: Option<Expr>, right: Expr) -> Expr {
match left {
Some(left) => Expr::BinaryExpr(BinaryExpr::new(
Box::new(left),
Operator::And,
Box::new(right),
)),
None => right,
}
}
/// The key and string values of a conjunct meaning "the key holds one of
/// these strings, or an array with one of them": the OR of scalar matches
/// and array-member matches over the same values and key.
fn string_membership(expr: &Expr) -> Option<(String, Vec<String>)> {
let mut disjuncts = Vec::new();
split_or(expr, &mut disjuncts);
let mut path: Option<&str> = None;
let mut scalars = BTreeSet::new();
let mut elements = BTreeSet::new();
for disjunct in disjuncts {
let (p, kind) = match unwrap_false_coalesce(disjunct) {
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Eq,
right,
}) => (
json_function(left, "json_extract")?,
Member::Scalar(vec![utf8(right)?]),
),
Expr::InList(list) if !list.negated => (
json_function(&list.expr, "json_extract")?,
Member::Scalar(list.list.iter().map(utf8).collect::<Option<_>>()?),
),
Expr::ScalarFunction(f) if f.func.name() == "json_array_contains" => {
let [data, p, value] = f.args.as_slice() else {
return None;
};
if !is_metadata(data) {
return None;
}
let element: serde_json::Value = serde_json::from_str(utf8(value)?).ok()?;
(utf8(p)?, Member::Element(element.as_str()?.to_string()))
}
_ => return None,
};
if path.is_some_and(|q| q != p) {
return None;
}
path = Some(p);
match kind {
Member::Scalar(values) => scalars.extend(values.into_iter().map(str::to_string)),
Member::Element(value) => {
elements.insert(value);
}
}
}
let key = json_path_key(path?)?;
let rendered: BTreeSet<String> = elements
.iter()
.map(|s| jsonb_text(s))
.collect::<Option<_>>()?;
(!elements.is_empty() && rendered == scalars).then(|| (key, elements.into_iter().collect()))
}
enum Member<'a> {
Scalar(Vec<&'a str>),
Element(String),
}
fn split_or<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) {
match expr {
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Or,
right,
}) => {
split_or(left, out);
split_or(right, out);
}
Expr::Alias(alias) => split_or(&alias.expr, out),
other => out.push(other),
}
}
/// `x` for `COALESCE(x, false)` or its simplified `x IS NOT NULL AND x`;
/// either is false where `x` is null, which a top-level conjunct treats alike.
fn unwrap_false_coalesce(expr: &Expr) -> &Expr {
match expr {
Expr::ScalarFunction(f) if f.func.name() == "coalesce" => match f.args.as_slice() {
[x, Expr::Literal(ScalarValue::Boolean(Some(false)), _)] => x,
_ => expr,
},
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::And,
right,
}) => match left.as_ref() {
Expr::IsNotNull(x) if x == right => right,
_ => expr,
},
_ => expr,
}
}
fn utf8(expr: &Expr) -> Option<&str> {
match expr {
Expr::Literal(ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)), _) => {
Some(s.as_str())
}
_ => None,
}
}
fn is_metadata(expr: &Expr) -> bool {
matches!(expr, Expr::Column(c) if c.name == METADATA_COLUMN)
}
/// The JSONPath literal of `name(metadata, path)`.
fn json_function<'a>(expr: &'a Expr, name: &str) -> Option<&'a str> {
match expr {
Expr::ScalarFunction(f) if f.func.name() == name => match f.args.as_slice() {
[data, path] if is_metadata(data) => utf8(path),
_ => None,
},
_ => None,
}
}
/// The key of a `$["key"]` path, with `\\` and `\"` unescaped.
fn json_path_key(path: &str) -> Option<String> {
let inner = path.strip_prefix("$[\"")?.strip_suffix("\"]")?;
let mut key = String::with_capacity(inner.len());
let mut chars = inner.chars();
while let Some(c) = chars.next() {
match c {
'\\' => match chars.next()? {
c @ ('\\' | '"') => key.push(c),
_ => return None,
},
'"' => return None,
c => key.push(c),
}
}
Some(key)
}
/// What json_extract returns for a stored string `s`.
fn jsonb_text(s: &str) -> Option<String> {
let value: OwnedJsonb = serde_json::to_string(s).ok()?.parse().ok()?;
Some(value.to_string())
}
/// How a filter compares a key.
pub mod operand {
pub const STRING: u8 = 1;
pub const NUMBER: u8 = 2;
pub const BOOL: u8 = 4;
pub const EXISTS: u8 = 8;
}
/// The metadata keys `filter` reads, each with the [`operand`] kinds it is
/// compared with.
pub fn filter_keys(dataset: &Dataset, filter: &str) -> Result<BTreeMap<String, u8>> {
let mut scanner = dataset.scan();
scanner.filter(filter)?;
let mut keys = BTreeMap::new();
if let Some(expr) = scanner.get_expr_filter()? {
collect_keys(&expr, &mut keys);
}
Ok(keys)
}
fn literal_kind(text: &str) -> u8 {
match serde_json::from_str::<serde_json::Value>(text) {
Ok(serde_json::Value::String(_)) => operand::STRING,
Ok(serde_json::Value::Number(_)) => operand::NUMBER,
Ok(serde_json::Value::Bool(_)) => operand::BOOL,
// json_extract's text of a string with control characters is not
// JSON, but it is quoted.
_ if text.starts_with('"') => operand::STRING,
_ => 0,
}
}
fn collect_keys(expr: &Expr, keys: &mut BTreeMap<String, u8>) {
let mut note = |path: &str, kind: u8| {
if let Some(key) = json_path_key(path) {
*keys.entry(key).or_default() |= kind;
}
};
match expr {
Expr::TryCast(cast) if extract(&cast.expr).is_some() => {
note(extract(&cast.expr).unwrap_or_default(), operand::NUMBER);
return;
}
Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
if let (Some(path), Some(value)) = (extract(left), utf8(right)) {
note(path, literal_kind(value));
return;
}
}
Expr::InList(list) => {
if let Some(path) = extract(&list.expr) {
let kind = list
.list
.iter()
.filter_map(utf8)
.fold(0, |k, v| k | literal_kind(v));
note(path, kind);
return;
}
}
Expr::ScalarFunction(f) if is_json_function(f.func.name()) => {
if let [data, path, rest @ ..] = f.args.as_slice()
&& is_metadata(data)
&& let Some(path) = utf8(path)
{
let kind = match (f.func.name(), rest) {
("json_exists", _) => operand::EXISTS,
(_, [value]) => utf8(value).map_or(0, literal_kind),
_ => 0,
};
note(path, kind);
return;
}
}
_ => {}
}
for child in children(expr) {
collect_keys(child, keys);
}
}
fn extract(expr: &Expr) -> Option<&str> {
json_function(expr, "json_extract")
}
fn is_json_function(name: &str) -> bool {
matches!(name, "json_extract" | "json_exists" | "json_array_contains")
}
fn children(expr: &Expr) -> Vec<&Expr> {
use lance::deps::datafusion::common::tree_node::TreeNode;
let mut out = Vec::new();
let _ = expr.apply_children(|child| {
out.push(child);
Ok(lance::deps::datafusion::common::tree_node::TreeNodeRecursion::Continue)
});
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::query::{self, Query};
use crate::table::VectorTable;
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use lance::index::DatasetIndexExt;
use lance_index::IndexType;
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
use lance_linalg::distance::DistanceType;
use std::sync::Arc;
use tempfile::TempDir;
const DIM: usize = 4;
/// The gateway's text for `{"author": "<v>"}`.
fn eq(field: &str, json: &str) -> String {
let json = json.replace('\'', "''");
format!(
"(COALESCE(json_extract(`metadata`, '$[\"{field}\"]') = '{json}', false) OR COALESCE(json_array_contains(`metadata`, '$[\"{field}\"]', '{json}'), false))"
)
}
/// The gateway's text for `{"author": {"$in": [<strings>]}}`.
fn among(values: &[&str]) -> String {
let scalars: Vec<String> = values.iter().map(|v| format!("'\"{v}\"'")).collect();
let mut groups = vec![format!(
"COALESCE(json_extract(`metadata`, '$[\"author\"]') in ({}), false)",
scalars.join(", ")
)];
for v in values {
groups.push(format!(
"COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"{v}\"'), false)"
));
}
format!("({})", groups.join(" OR "))
}
/// Rows `u0..u<n>` by author, plus a few others, in fragments of 100,
/// with the author column ready and indexed and a vector index.
async fn dataset(dir: &TempDir, rows: usize) -> Dataset {
let metadata = Field::new("metadata", DataType::Utf8, true)
.with_metadata([("ARROW:extension:name".to_string(), "arrow.json".to_string())].into());
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
false,
),
metadata,
metadata_index::field("_mk_0", "author", State::Ready),
]));
let metas: Vec<serde_json::Value> = (0..rows)
.map(|i| match i % 10 {
0 => serde_json::json!({"author": "rare"}),
1 => serde_json::json!({"author": ["many", "rare2"]}),
_ => serde_json::json!({"author": "many", "n": i}),
})
.collect();
let batches: Vec<RecordBatch> = (0..rows)
.step_by(100)
.map(|start| {
let range = start..(start + 100).min(rows);
RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(StringArray::from(
range.clone().map(|i| format!("k{i}")).collect::<Vec<_>>(),
)),
Arc::new(FixedSizeListArray::from_iter_primitive::<
arrow_array::types::Float32Type,
_,
_,
>(
range.clone().map(|i| {
Some((0..DIM).map(move |j| Some(((i * 31 + j * 17) % 97) as f32)))
}),
DIM as i32,
)),
Arc::new(StringArray::from(
range
.clone()
.map(|i| Some(metas[i].to_string()))
.collect::<Vec<_>>(),
)),
metadata_index::column_from_json(range.map(|i| Some(&metas[i])), "author"),
],
)
.unwrap()
})
.collect();
let uri = dir.path().to_str().unwrap();
let mut dataset = None::<Dataset>;
for batch in batches {
let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
dataset = Some(match dataset {
None => Dataset::write(reader, uri, None).await.unwrap(),
Some(mut d) => {
d.append(reader, None).await.unwrap();
d
}
});
}
let mut dataset = dataset.unwrap();
dataset
.create_index(
&["_mk_0"],
IndexType::LabelList,
None,
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
true,
)
.await
.unwrap();
crate::index::build(&mut dataset, "vector", crate::index::INDEX_NAME, 4, false)
.await
.unwrap();
dataset
}
fn rewritten(dataset: &Dataset, filter: &str) -> (String, Option<String>) {
let mut scanner = dataset.scan();
let indexed = crate::filter::apply(&mut scanner, filter, &ready_columns(dataset)).unwrap();
(
scanner.get_expr_filter().unwrap().unwrap().to_string(),
indexed.map(|e| e.to_string()),
)
}
#[tokio::test]
async fn string_eq_and_in_on_a_ready_key_use_its_column() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 200).await;
assert_eq!(
ready_columns(&dataset),
HashMap::from([("author".to_string(), "_mk_0".to_string())])
);
for (filter, want) in [
(eq("author", "\"rare\""), Some("array_has(_mk_0, Utf8(\"rare\"))")),
(
among(&["rare", "many"]),
Some("array_has_any(_mk_0, List([many, rare]))"),
),
(
format!("{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3", eq("author", "\"rare\"")),
Some("array_has(_mk_0, Utf8(\"rare\"))"),
),
(eq("author", "\"o'neil \\\"q\\\"\""), Some("array_has(_mk_0, Utf8(\"o'neil \"q\"\"))")),
(
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u0001d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
Some("array_has(_mk_0, Utf8(\"c\u{1}d\n\"))"),
),
// The scalar half's text is not what json_extract returns for
// the array half's string, so the two do not agree.
(
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u{1}d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
None,
),
// Not a string, not this key, or not every value on both sides.
(eq("author", "true"), None),
(eq("other", "\"rare\""), None),
(format!("NOT {}", eq("author", "\"rare\"")), None),
(format!("NOT {}", among(&["rare", "many"])), None),
(
"COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"rare\"', false)".to_string(),
None,
),
(
among(&["rare", "many"]).replace("'\"many\"'), false))", "'\"other\"'), false))"),
None,
),
("json_exists(`metadata`, '$[\"author\"]')".to_string(), None),
(format!("({} OR json_exists(`metadata`, '$[\"n\"]'))", eq("author", "\"rare\"")), None),
] {
let (expr, indexed) = rewritten(&dataset, &filter);
assert_eq!(indexed.as_deref(), want, "{filter}");
assert_eq!(expr.contains("json_"), want.is_none() || filter.contains(" AND "), "{filter}: {expr}");
}
}
/// The planner scores few matches exactly and leaves many to the vector
/// index, per scope.
#[tokio::test]
async fn few_indexed_matches_are_scored_exactly() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 1000).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
let query = |filter: &str, fragments: Option<Vec<u64>>| Query {
vector: vec![1.0, 2.0, 3.0, 4.0],
k: 10,
filter: Some(filter.to_string()),
postfilter: false,
distance: DistanceType::L2,
nprobes: None,
index_segments: None,
fragments,
exact_match_limit: None,
};
let rare = eq("author", "\"rare\"");
let many = eq("author", "\"many\"");
let few = |q: Query| {
let (dataset, table) = (&dataset, &table);
async move {
query::few_indexed_matches(dataset, table, &q)
.await
.unwrap()
}
};
// 100 and 900 matches, and 90 in the first two fragments.
assert!(few(query(&rare, None)).await);
assert!(!few(query(&many, None)).await);
let first = Some(vec![0, 1]);
assert!(few(query(&many, first.clone())).await);
assert!(
!few(Query {
exact_match_limit: Some(0),
..query(&rare, None)
})
.await
);
// A JSON-only filter is not counted.
assert!(!few(query("json_exists(`metadata`, '$[\"n\"]')", None)).await);
for (filter, fragments) in [(&rare, None), (&many, first)] {
let q = query(filter, fragments);
let exact = query::flat(&dataset, &table, &q).await.unwrap();
let got = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(got.len(), 10);
for (g, e) in got.iter().zip(&exact) {
assert!(
(g.distance - e.distance).abs() < 1e-4,
"{got:?} vs {exact:?}"
);
}
}
}
#[tokio::test]
async fn the_plan_reads_the_scalar_index() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 300).await;
let mut scanner = dataset.scan();
crate::filter::apply(
&mut scanner,
&eq("author", "\"rare\""),
&ready_columns(&dataset),
)
.unwrap();
let plan = scanner.explain_plan(true).await.unwrap();
assert!(plan.contains("LabelList"), "{plan}");
assert!(!plan.contains("json_"), "{plan}");
}
#[tokio::test]
async fn filter_keys_note_how_each_key_is_compared() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 100).await;
let filter = format!(
"{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3 AND NOT json_exists(`metadata`, '$[\"a.b\\\"c\"]') AND COALESCE(json_extract(`metadata`, '$[\"f\"]') = 'true', false)",
eq("author", "\"rare\"")
);
let keys = filter_keys(&dataset, &filter).unwrap();
assert_eq!(
keys,
BTreeMap::from([
("a.b\"c".to_string(), operand::EXISTS),
("author".to_string(), operand::STRING),
("f".to_string(), operand::BOOL),
("n".to_string(), operand::NUMBER),
])
);
}
}
-1
View File
@@ -8,7 +8,6 @@
pub mod coverage;
mod filter;
pub mod index;
pub mod indexed_filter;
pub mod maintenance;
pub mod query;
pub mod segments;
-1
View File
@@ -309,7 +309,6 @@ async fn main() -> Result<()> {
nprobes,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let hits = if flat {
query::flat(&dataset, &vector_table, &query).await?
@@ -312,7 +312,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let mut ids: Vec<u64> = query::flat(&pinned, &resolved, &q)
.await
@@ -735,7 +734,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(query::flat(&dataset, &resolved, &q).await.is_err());
}
+6 -145
View File
@@ -39,15 +39,8 @@ pub struct Query {
/// a worker the unindexed-fragment sweep. Listed ids that no longer exist
/// scan nothing.
pub fragments: Option<Vec<u64>>,
/// Score the matches exactly instead of searching the vector index when
/// the filter's indexed metadata keys match at most this many rows of the
/// scope; 0 never does. None takes [`DEFAULT_EXACT_MATCH_LIMIT`].
pub exact_match_limit: Option<usize>,
}
/// Up to this many matches of an indexed metadata key are scored exactly.
pub const DEFAULT_EXACT_MATCH_LIMIT: usize = 256;
/// One row of a top-K answer.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Hit {
@@ -60,103 +53,14 @@ pub struct Hit {
/// Exact top-K over every fragment, ignoring any index.
pub async fn flat(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
run(scanner(dataset, table, query, false).await?.0, table).await
run(scanner(dataset, table, query, false).await?, table).await
}
/// Top-K with the ANN index where it covers the fragment and a flat pass over
/// the rest. Whether the index is used is Lance's decision per fragment, so
/// rows written after the last index build are always candidates.
///
/// A prefiltered search sees only the probed partitions: it can come back
/// short of k, or with unscored rows (infinite distance) when at most k rows
/// match. Either way it is answered again by an exact scan of the same scope.
///
/// When the filter's indexed metadata keys match few rows of that scope,
/// they are scored exactly to begin with. The decision is per scope, so
/// exact answers of several scopes merge into the exact answer.
pub async fn ann(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
if few_indexed_matches(dataset, table, query).await? {
return exact_in_scope(dataset, table, query).await;
}
let (scanner, indexed) = scanner(dataset, table, query, true).await?;
let hits = run(scanner, table).await?;
if indexed
&& query.filter.is_some()
&& !query.postfilter
&& (hits.len() < query.k || hits.iter().any(|h| !h.distance.is_finite()))
{
return exact_in_scope(dataset, table, query).await;
}
Ok(hits)
}
/// [`flat`] over what [`ann`] searches for `query`: the listed fragments plus
/// the live coverage of the listed index segments, or the whole table.
async fn exact_in_scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
let exact = Query {
index_segments: None,
fragments: scope(dataset, table, query).await?,
..query.clone()
};
run(scanner(dataset, table, &exact, false).await?.0, table).await
}
/// The fragments [`ann`] searches for `query`, or None for the whole table.
async fn scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Option<Vec<u64>>> {
Ok(match &query.index_segments {
Some(segments) if !segments.is_empty() => {
let mut ids = query.fragments.clone().unwrap_or_default();
ids.extend(
crate::segments::covered_fragments(dataset, &table.vector_column, segments).await?,
);
Some(ids)
}
_ => query.fragments.clone(),
})
}
/// Whether the rows of [`ann`]'s scope matching the filter's indexed
/// metadata keys, counted through their scalar indexes, are at most the
/// query's limit. Only a search through the vector index asks.
pub(crate) async fn few_indexed_matches(
dataset: &Dataset,
table: &VectorTable,
query: &Query,
) -> Result<bool> {
let limit = query.exact_match_limit.unwrap_or(DEFAULT_EXACT_MATCH_LIMIT);
let Some(filter) = &query.filter else {
return Ok(false);
};
if limit == 0 || query.postfilter || query.index_segments.as_ref().is_some_and(|s| s.is_empty())
{
return Ok(false);
}
let ready = crate::indexed_filter::ready_columns(dataset);
if ready.is_empty()
|| crate::index::index_names(dataset, &table.vector_column)
.await?
.is_empty()
{
return Ok(false);
}
let Some(indexed) = crate::filter::apply(&mut dataset.scan(), filter, &ready)? else {
return Ok(false);
};
let mut counter = dataset.scan();
counter.filter_expr(indexed);
if let Some(ids) = scope(dataset, table, query).await? {
counter.with_fragments(
dataset
.fragments()
.iter()
.filter(|f| ids.contains(&f.id))
.cloned()
.collect(),
);
}
let matches = counter.count_rows().await?;
tracing::debug!(matches, limit, "indexed metadata filter");
Ok(matches <= limit as u64)
run(scanner(dataset, table, query, true).await?, table).await
}
/// Fetch rows by key, projected to `columns` (all of them when None).
@@ -197,7 +101,7 @@ async fn scanner(
table: &VectorTable,
query: &Query,
use_index: bool,
) -> Result<(Scanner, bool)> {
) -> Result<Scanner> {
if query.vector.len() != table.dimension {
bail!(
"query dim({}) does not match the table's dim({})",
@@ -213,24 +117,18 @@ async fn scanner(
.use_index(use_index)
.with_row_id();
if let Some(filter) = &query.filter {
let ready = crate::indexed_filter::ready_columns(dataset);
crate::filter::apply(&mut scanner, filter, &ready)?;
crate::filter::apply(&mut scanner, filter)?;
scanner.prefilter(!query.postfilter);
}
// An explicitly empty segment list searches no index, so there is
// nothing to size probes for.
let no_segments = query.index_segments.as_ref().is_some_and(|s| s.is_empty());
let mut indexed = false;
if use_index && !no_segments {
let probes = match query.nprobes {
Some(n) => crate::index::index_names(dataset, &table.vector_column)
.await?
.first()
.map(|_| crate::index::Probes { min: n, max: n }),
Some(n) => Some(crate::index::Probes { min: n, max: n }),
None => crate::index::default_probes(dataset, &table.vector_column).await?,
};
if let Some(probes) = probes {
indexed = true;
// Lance searches `min` partitions, and under a prefilter keeps
// going up to `max` until it has k rows. An open bound would
// make it prepare every partition for that, which costs as much
@@ -280,7 +178,7 @@ async fn scanner(
columns.push(DISTANCE_COLUMN.to_string());
columns.push(ROW_ID_COLUMN.to_string());
scanner.project(&columns)?;
Ok((scanner, indexed))
Ok(scanner)
}
async fn run(scanner: Scanner, table: &VectorTable) -> Result<Vec<Hit>> {
@@ -363,7 +261,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let hits = flat(&dataset, &table, &query).await.unwrap();
assert_eq!(hits.len(), 2);
@@ -405,7 +302,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -485,7 +381,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -589,7 +484,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(flat(&dataset, &table, &query).await.is_err());
}
@@ -613,7 +507,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let indexed = ann(&dataset, &table, &query).await.unwrap();
let exact = flat(&dataset, &table, &query).await.unwrap();
@@ -622,36 +515,4 @@ mod tests {
exact.iter().map(|h| &h.key).collect::<Vec<_>>()
);
}
/// An explicit probe count does not make an unindexed table's scan count
/// as indexed, so a short filtered answer is not scanned a second time.
#[tokio::test]
async fn explicit_probes_need_an_index() {
let dir = TempDir::new().unwrap();
let rows = crate::testing::vectors(300, DIM);
let mut dataset = write_dataset(&dir, &rows, &["a"; 300]).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
let query = Query {
vector: rows[0].clone(),
k: 10,
filter: Some("tag = 'a'".to_string()),
postfilter: false,
distance: DistanceType::L2,
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(!scanner(&dataset, &table, &query, true).await.unwrap().1);
crate::index::build(
&mut dataset,
&table.vector_column,
crate::index::INDEX_NAME,
4,
false,
)
.await
.unwrap();
assert!(scanner(&dataset, &table, &query, true).await.unwrap().1);
}
}
@@ -139,7 +139,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let want: Vec<(u64, u32)> = query::ann(&dataset, &resolved, &base)
.await
@@ -240,7 +239,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: Some(vec![u64::MAX]),
exact_match_limit: None,
};
assert!(
query::ann(&dataset, &resolved, &q)
-1
View File
@@ -44,7 +44,6 @@ fn query_for(vector: Vec<f32>, k: usize) -> Query {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
}
}
-1
View File
@@ -46,7 +46,6 @@ tracing.workspace = true
tracing-subscriber.workspace = true
[dev-dependencies]
lance-index = "10"
tempfile = "3"
[build-dependencies]
@@ -1,290 +0,0 @@
//! How often, and how slowly, each index's queries filter on each metadata
//! key over the last hour. Keys come from the parsed filter; the number of
//! indexes and of keys per index is bounded, so arbitrary keys cannot grow
//! memory without limit. Not exported as metrics: a key is no label.
use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use std::time::{Duration, Instant};
use weed_vector_core::indexed_filter::operand;
use crate::pb::vector as pb;
const BUCKETS: usize = 12;
const BUCKET: Duration = Duration::from_secs(300);
pub const WINDOW: Duration = Duration::from_secs(BUCKET.as_secs() * BUCKETS as u64);
pub const MAX_INDEXES: usize = 512;
pub const MAX_KEYS_PER_INDEX: usize = 32;
#[derive(Default, Clone, Copy)]
struct Bucket {
/// The bucket's number since the stats started.
at: u64,
queries: u64,
seconds: f64,
/// The operand kinds the bucket's queries compared the key with.
kinds: u8,
}
#[derive(Default)]
struct KeyStats {
buckets: [Bucket; BUCKETS],
}
#[derive(Default)]
struct IndexStats {
keys: HashMap<String, KeyStats>,
/// The last bucket a key was left uncounted in.
truncated: Option<u64>,
last: u64,
}
/// Whether bucket `at` is still in the window of bucket `now`.
fn live(at: u64, now: u64) -> bool {
at + BUCKETS as u64 > now
}
pub struct FilterStats {
start: Instant,
indexes: Mutex<HashMap<String, IndexStats>>,
}
impl Default for FilterStats {
fn default() -> Self {
Self::new()
}
}
impl FilterStats {
pub fn new() -> Self {
Self {
start: Instant::now(),
indexes: Mutex::new(HashMap::new()),
}
}
fn bucket(&self, now: Instant) -> u64 {
(now.duration_since(self.start).as_secs() / BUCKET.as_secs()) + 1
}
/// One query of `index` that took `elapsed` and filtered on `keys`.
pub fn record(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration) {
self.record_at(index, keys, elapsed, Instant::now());
}
fn record_at(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration, now: Instant) {
if keys.is_empty() {
return;
}
let at = self.bucket(now);
let mut indexes = self.indexes.lock().unwrap();
if !indexes.contains_key(index) && indexes.len() >= MAX_INDEXES {
let Some(oldest) = indexes
.iter()
.min_by_key(|(_, stats)| stats.last)
.map(|(name, _)| name.clone())
else {
return;
};
indexes.remove(&oldest);
}
let stats = indexes.entry(index.to_string()).or_default();
stats.last = stats.last.max(at);
for (key, kinds) in keys {
if !stats.keys.contains_key(key) && stats.keys.len() >= MAX_KEYS_PER_INDEX {
// Make room only by dropping a key no query used in the window.
let idle = stats
.keys
.iter()
.find(|(_, k)| k.buckets.iter().all(|b| !live(b.at, at)))
.map(|(name, _)| name.clone());
match idle {
Some(idle) => {
stats.keys.remove(&idle);
}
None => {
stats.truncated = stats.truncated.max(Some(at));
continue;
}
}
}
let entry = stats.keys.entry(key.clone()).or_default();
let bucket = &mut entry.buckets[(at % BUCKETS as u64) as usize];
if bucket.at != at {
*bucket = Bucket {
at,
..Default::default()
};
}
bucket.queries += 1;
bucket.seconds += elapsed.as_secs_f64();
bucket.kinds |= kinds;
}
}
/// The stats of `index` over the window.
pub fn snapshot(&self, index: &str) -> pb::FilterStatsResponse {
self.snapshot_at(index, Instant::now())
}
fn snapshot_at(&self, index: &str, now: Instant) -> pb::FilterStatsResponse {
let at = self.bucket(now);
let window_seconds = now.duration_since(self.start).min(WINDOW).as_secs() as u32;
let indexes = self.indexes.lock().unwrap();
let Some(stats) = indexes.get(index) else {
return pb::FilterStatsResponse {
window_seconds,
..Default::default()
};
};
let mut keys: Vec<pb::FilterKeyStats> = stats
.keys
.iter()
.filter_map(|(key, s)| {
let (queries, seconds, kinds) = s
.buckets
.iter()
.filter(|b| live(b.at, at))
.fold((0, 0.0, 0), |(q, t, k), b| {
(q + b.queries, t + b.seconds, k | b.kinds)
});
(queries > 0).then(|| pb::FilterKeyStats {
key: key.clone(),
queries,
seconds,
string_operand: kinds & operand::STRING != 0,
number_operand: kinds & operand::NUMBER != 0,
bool_operand: kinds & operand::BOOL != 0,
})
})
.collect();
keys.sort_by(|a, b| b.queries.cmp(&a.queries).then(a.key.cmp(&b.key)));
pb::FilterStatsResponse {
keys,
window_seconds,
truncated: stats.truncated.is_some_and(|t| live(t, at)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn keys(pairs: &[(&str, u8)]) -> BTreeMap<String, u8> {
pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
}
#[test]
fn counts_queries_and_latency_per_key_within_the_window() {
let stats = FilterStats::new();
let t0 = stats.start;
let ms = Duration::from_millis;
stats.record_at("b/i", &keys(&[("author", operand::STRING)]), ms(10), t0);
stats.record_at(
"b/i",
&keys(&[("author", operand::STRING), ("year", operand::NUMBER)]),
ms(30),
t0 + Duration::from_secs(600),
);
stats.record_at("b/j", &keys(&[("x", operand::BOOL)]), ms(5), t0);
let got = stats.snapshot_at("b/i", t0 + Duration::from_secs(900));
assert_eq!(got.window_seconds, 900);
assert_eq!(got.keys.len(), 2);
assert_eq!(got.keys[0].key, "author");
assert_eq!(got.keys[0].queries, 2);
assert!((got.keys[0].seconds - 0.04).abs() < 1e-9);
assert!(got.keys[0].string_operand && !got.keys[0].number_operand);
assert_eq!(got.keys[1].key, "year");
assert!(got.keys[1].number_operand);
// The first query leaves the window an hour after its bucket.
let later = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(1));
assert_eq!(later.window_seconds, WINDOW.as_secs() as u32);
assert_eq!(later.keys[0].queries, 1);
let year = later.keys.iter().find(|k| k.key == "year").unwrap();
assert!(year.number_operand);
// A string query that left the window no longer marks the key.
stats.record_at(
"b/i",
&keys(&[("author", operand::NUMBER)]),
ms(1),
t0 + WINDOW + Duration::from_secs(2),
);
let mixed = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(3));
let author = mixed.keys.iter().find(|k| k.key == "author").unwrap();
assert!(author.string_operand && author.number_operand);
let numeric = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(700));
let author = numeric.keys.iter().find(|k| k.key == "author").unwrap();
assert!(!author.string_operand && author.number_operand);
let gone = stats.snapshot_at("b/i", t0 + 2 * WINDOW + Duration::from_secs(900));
assert!(gone.keys.is_empty());
assert!(stats.snapshot_at("other", t0).keys.is_empty());
}
#[test]
fn indexes_and_keys_are_bounded() {
let stats = FilterStats::new();
let t0 = stats.start;
for i in 0..MAX_KEYS_PER_INDEX + 5 {
stats.record_at(
"b/i",
&keys(&[(&format!("k{i}"), operand::STRING)]),
Duration::ZERO,
t0,
);
}
let got = stats.snapshot_at("b/i", t0);
assert_eq!(got.keys.len(), MAX_KEYS_PER_INDEX);
assert!(got.truncated);
// A key idle for the whole window makes room for a new one.
let later = t0 + WINDOW + BUCKET;
stats.record_at(
"b/i",
&keys(&[("new", operand::STRING)]),
Duration::ZERO,
later,
);
let got = stats.snapshot_at("b/i", later);
assert!(!got.truncated, "the uncounted keys left the window");
assert_eq!(got.keys.len(), 1);
assert_eq!(got.keys[0].key, "new");
for i in 0..MAX_INDEXES + 3 {
stats.record_at(
&format!("b/{i}"),
&keys(&[("k", operand::STRING)]),
Duration::ZERO,
later,
);
}
assert_eq!(stats.indexes.lock().unwrap().len(), MAX_INDEXES);
}
#[test]
fn a_record_from_an_earlier_bucket_keeps_the_later_truncation() {
let stats = FilterStats::new();
let t0 = stats.start;
for i in 0..=MAX_KEYS_PER_INDEX {
stats.record_at(
"b/i",
&keys(&[(&format!("k{i}"), operand::STRING)]),
Duration::ZERO,
t0 + BUCKET,
);
}
// Took its time before the lock while the later bucket recorded.
stats.record_at(
"b/i",
&keys(&[("late", operand::STRING)]),
Duration::ZERO,
t0,
);
let got = stats.snapshot_at("b/i", t0 + WINDOW);
assert!(
got.truncated,
"the later bucket's uncounted key is in the window"
);
assert_eq!(stats.indexes.lock().unwrap()["b/i"].last, 2);
}
}
@@ -1,627 +0,0 @@
//! Indexed metadata keys through the VectorService: PutVectors derives the
//! columns, and a query answers the same with or without them.
use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_schema::Schema;
use lance::Dataset;
use lance::dataset::{BatchUDF, NewColumnTransform};
use lance::index::DatasetIndexExt;
use lance_index::IndexType;
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
use tonic::Request;
use weed_lance_worker::metadata_index::{self, STATE_FIELD_META, State};
use weed_vector_core::query::{self, Query};
use weed_vector_core::table::VectorTable;
use crate::pb::vector::{self as pb, vector_service_server::VectorService};
use crate::service::VectorServer;
const DIM: usize = 8;
const ROWS: usize = 3000;
const COLUMN: &str = "_mk_0";
/// How the engine answers a filter once `author` is indexed.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Path {
/// Through the column, at most the exact-match limit of rows: exact.
Exact,
/// Through the column, more rows: the vector index with the column's
/// prefilter.
Ivf,
/// Not through the column.
Json,
}
/// S3 Vectors filters, the gateway's translation of each, and its path.
const FILTERS: [(&str, &str, Path); 22] = [
(
"{\"author\":\"alice\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
Path::Ivf,
),
(
"{\"author\":{\"$eq\":\"bob\"}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
Path::Exact,
),
(
"{\"author\":\"carol\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"carol\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"carol\"'), false))",
Path::Exact,
),
(
"{\"author\":\"x\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
Path::Exact,
),
(
"{\"author\":\"7\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"7\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"7\"'), false))",
Path::Exact,
),
(
"{\"author\":7}",
"(COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) = 7, false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
Path::Json,
),
(
"{\"author\":true}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = 'true', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', 'true'), false))",
Path::Json,
),
(
"{\"author\":{\"$in\":[\"alice\",\"bob\",\"x\"]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"', '\"x\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
Path::Ivf,
),
(
"{\"author\":{\"$in\":[\"alice\",7]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"'), false) OR COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) in (7), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
Path::Json,
),
(
"{\"author\":{\"$ne\":\"alice\"}}",
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$nin\":[\"alice\",\"bob\"]}}",
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$exists\":false}}",
"NOT json_exists(`metadata`, '$[\"author\"]')",
Path::Json,
),
(
"{\"$and\":[{\"author\":\"alice\"},{\"genre\":\"g1\"}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false)) AND (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false)))",
Path::Ivf,
),
(
"{\"$and\":[{\"author\":\"bob\"},{\"year\":{\"$gt\":2005}}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) AND TRY_CAST(json_extract(`metadata`, '$[\"year\"]') AS DOUBLE) > 2005)",
Path::Exact,
),
(
"{\"$or\":[{\"author\":\"bob\"},{\"genre\":\"g2\"}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) OR (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g2\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g2\"'), false)))",
Path::Json,
),
(
"{\"author\":\"o'neil \\\"q\\\" \\\\ \\n\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"o''neil \\\"q\\\" \\\\ \\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"o''neil \\\"q\\\" \\\\ \\n\"'), false))",
Path::Exact,
),
(
"{\"author\":\"nobody\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"nobody\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"nobody\"'), false))",
Path::Exact,
),
("{\"author\":{\"$in\":[]}}", "false", Path::Json),
(
"{\"genre\":\"g1\"}",
"(COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false))",
Path::Json,
),
(
"{\"author\":\"u5\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"u5\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false))",
Path::Exact,
),
(
"{\"author\":\"ctl\\u0001x\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"ctl\\u{1}x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"ctl\\u0001x\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$in\":[\"u5\",\"u6\"]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"u5\"', '\"u6\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u6\"'), false))",
Path::Exact,
),
];
fn rng(seed: &mut u64) -> f32 {
*seed = seed
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
((*seed >> 33) as f32 / (1u64 << 31) as f32) - 0.5
}
fn vector_of(i: usize) -> Vec<f32> {
let mut centre = (i % 16) as u64 + 1;
let mut noise = i as u64 * 7919 + 13;
(0..DIM)
.map(|_| rng(&mut centre) * 4.0 + rng(&mut noise))
.collect()
}
fn author(i: usize) -> Option<serde_json::Value> {
use serde_json::json;
Some(match i % 100 {
0..=9 => json!("alice"),
10 => json!("bob"),
11 => json!(["carol", "x"]),
12 => json!(7),
13 => json!("7"),
14 => serde_json::Value::Null,
15 => return None,
16 => json!(true),
17 => json!("o'neil \"q\" \\ \n"),
18 => json!("ctl\u{1}x"),
19 => json!(["alice", 3]),
_ => json!(format!("u{}", i % 37)),
})
}
fn row(i: usize, author: Option<serde_json::Value>) -> pb::Vector {
let mut meta = serde_json::json!({"genre": format!("g{}", i % 3), "year": 2000 + i % 10});
if let Some(author) = author {
meta["author"] = author;
}
let serde_json::Value::Object(meta) = meta else {
unreachable!()
};
pb::Vector {
key: format!("k{i}"),
data: vector_of(i),
metadata: Some(crate::service::json_to_struct(meta)),
}
}
struct Fixture {
server: VectorServer,
exact_off: VectorServer,
table: pb::Table,
}
impl Fixture {
async fn new() -> Self {
let bucket = format!(
"mk-{:x}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let table = pb::Table {
id: vec![bucket.clone(), "vector".into(), "x".into()],
location: format!("shared-memory://{bucket}/vector/x"),
..Default::default()
};
let fixture = Self {
server: VectorServer::new().with_schemes(&["shared-memory"]),
exact_off: VectorServer::new()
.with_schemes(&["shared-memory"])
.with_exact_match_limit(Some(0)),
table,
};
fixture
.server
.create_index(Request::new(pb::CreateIndexRequest {
table: Some(fixture.table.clone()),
dimension: DIM as u32,
distance_metric: pb::DistanceMetric::L2 as i32,
..Default::default()
}))
.await
.unwrap();
fixture
}
async fn put(&self, vectors: Vec<pb::Vector>) {
self.server
.put_vectors(Request::new(pb::PutVectorsRequest {
table: Some(self.table.clone()),
vectors,
}))
.await
.unwrap();
}
async fn dataset(&self, version: u64) -> Dataset {
let dataset = Dataset::open(&self.table.location).await.unwrap();
if version == 0 {
dataset
} else {
dataset.checkout_version(version).await.unwrap()
}
}
/// Adds `author`'s column, indexes it and marks it ready, returning the
/// version before it was ready.
async fn index_author(&self) -> u64 {
let mut dataset = self.dataset(0).await;
let output = Arc::new(Schema::new(vec![metadata_index::field(
COLUMN,
"author",
State::Building,
)]));
let schema = output.clone();
dataset
.add_columns(
NewColumnTransform::BatchUDF(BatchUDF {
mapper: Box::new(move |batch: &RecordBatch| {
let column = metadata_index::column_from_metadata(
batch.column_by_name("metadata").unwrap().as_ref(),
"author",
)
.map_err(|e| lance::Error::invalid_input(e.to_string()))?;
Ok(RecordBatch::try_new(schema.clone(), vec![column])?)
}),
output_schema: output,
result_checkpoint: None,
}),
Some(vec!["metadata".to_string()]),
None,
)
.await
.unwrap();
dataset
.create_index(
&[COLUMN],
IndexType::LabelList,
Some(format!("{COLUMN}_idx")),
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
true,
)
.await
.unwrap();
let building = dataset.version().version;
self.set_state(State::Ready).await;
building
}
async fn set_state(&self, state: State) -> u64 {
let mut dataset = self.dataset(0).await;
dataset
.update_field_metadata()
.update(COLUMN, [(STATE_FIELD_META, state.as_str())])
.unwrap()
.await
.unwrap();
dataset.version().version
}
async fn query(
&self,
server: &VectorServer,
version: u64,
filter: &str,
k: u32,
v: &[f32],
) -> Vec<(String, f32)> {
let mut table = self.table.clone();
table.version = version;
server
.query_vectors(Request::new(pb::QueryVectorsRequest {
table: Some(table),
query_vector: v.to_vec(),
top_k: k,
filter: filter.to_string(),
return_distance: true,
..Default::default()
}))
.await
.unwrap()
.into_inner()
.vectors
.into_iter()
.map(|r| (r.key, r.distance))
.collect()
}
/// Exact filtered top-k at `version`.
async fn flat(&self, version: u64, filter: &str, k: u32, v: &[f32]) -> Vec<(String, f32)> {
let dataset = self.dataset(version).await;
let table = VectorTable::resolve(&dataset, None, Some("key")).unwrap();
query::flat(
&dataset,
&table,
&Query {
vector: v.to_vec(),
k: k as usize,
filter: Some(filter.to_string()),
postfilter: false,
distance: lance_linalg::distance::DistanceType::L2,
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
.unwrap()
.into_iter()
.map(|h| (h.key, h.distance))
.collect()
}
/// Every filter answers through the column as through the JSON at
/// `json` (the same rows and indexes, the column not ready), and an
/// exact answer as the exact filtered top-k.
async fn compare(&self, json: u64, ready: u64) {
for q in [7, 1234] {
let v = vector_of(q);
for (s3, filter, path) in FILTERS {
for k in [10, 400] {
let what = format!("{s3} k={k} q={q}");
let exact = self.flat(json, filter, k, &v).await;
assert_same(&self.flat(ready, filter, k, &v).await, &exact, &what);
let json_ann = self.query(&self.server, json, filter, k, &v).await;
let off = self.query(&self.exact_off, ready, filter, k, &v).await;
assert_same(&off, &json_ann, &format!("{what}: indexed prefilter"));
let got = self.query(&self.server, ready, filter, k, &v).await;
let want = if path == Path::Exact {
&exact
} else {
&json_ann
};
assert_same(&got, want, &format!("{what}: {path:?}"));
}
}
}
}
}
fn assert_same(got: &[(String, f32)], want: &[(String, f32)], what: &str) {
assert_eq!(got.len(), want.len(), "{what}: {got:?} vs {want:?}");
for (g, w) in got.iter().zip(want) {
let same = g.0 == w.0 || (g.1 - w.1).abs() < 1e-4;
assert!(
same && ((g.1 - w.1).abs() < 1e-4 || (g.1.is_infinite() && w.1.is_infinite())),
"{what}: {got:?} vs {want:?}"
);
}
}
/// Indexed and JSON answers agree on mixed types, missing keys, arrays,
/// null, untouched operators, and after updates, deletes and rows no
/// scalar or vector index covers yet.
#[tokio::test]
async fn indexed_answers_match_the_json_path() {
let fixture = Fixture::new().await;
for start in (0..ROWS).step_by(300) {
fixture
.put((start..start + 300).map(|i| row(i, author(i))).collect())
.await;
}
let mut dataset = fixture.dataset(0).await;
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
.await
.unwrap();
let json = fixture.index_author().await;
let schema: Schema = fixture.dataset(0).await.schema().into();
assert_eq!(
metadata_index::indexed_keys(&schema)[0].state,
Some(State::Ready)
);
fixture.compare(json, 0).await;
// Updates move rows between values, deletes drop some, and new rows land
// in fragments neither index covers; PutVectors fills the column.
fixture
.put(
(0..60)
.map(|i| row(i, Some(serde_json::json!(["moved", "bob"]))))
.chain((ROWS..ROWS + 200).map(|i| row(i, author(i))))
.collect(),
)
.await;
fixture
.server
.delete_vectors(Request::new(pb::DeleteVectorsRequest {
table: Some(fixture.table.clone()),
keys: (100..130).map(|i| format!("k{i}")).collect(),
}))
.await
.unwrap();
let json = fixture.set_state(State::Building).await;
let ready = fixture.set_state(State::Ready).await;
fixture.compare(json, ready).await;
let moved = r#"(COALESCE(json_extract(`metadata`, '$["author"]') = '"moved"', false) OR COALESCE(json_array_contains(`metadata`, '$["author"]', '"moved"'), false))"#;
let hits = fixture
.query(&fixture.server, ready, moved, 100, &vector_of(0))
.await;
assert_eq!(hits.len(), 60);
}
/// The planner counts and scores per scope: an exact answer over a fragment
/// subset is that subset's exact top-k.
#[tokio::test]
async fn a_scoped_query_is_exact_within_its_fragments() {
let fixture = Fixture::new().await;
for start in (0..ROWS).step_by(300) {
fixture
.put((start..start + 300).map(|i| row(i, author(i))).collect())
.await;
}
let mut dataset = fixture.dataset(0).await;
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
.await
.unwrap();
let json = fixture.index_author().await;
let ready = fixture.dataset(0).await;
let table = VectorTable::resolve(&ready, None, Some("key")).unwrap();
let ids: Vec<u64> = ready.fragments().iter().map(|f| f.id).take(4).collect();
let filter = FILTERS[1].1;
let scoped = || Query {
vector: vector_of(5),
k: 10,
filter: Some(filter.to_string()),
postfilter: false,
distance: lance_linalg::distance::DistanceType::L2,
nprobes: None,
index_segments: None,
fragments: Some(ids.clone()),
exact_match_limit: None,
};
let got = query::ann(&ready, &table, &scoped()).await.unwrap();
let old = fixture.dataset(json).await;
let want = query::flat(&old, &table, &scoped()).await.unwrap();
assert_eq!(want.len(), 10);
let pairs = |hits: Vec<weed_vector_core::Hit>| {
hits.into_iter()
.map(|h| (h.key, h.distance))
.collect::<Vec<_>>()
};
assert_same(&pairs(got), &pairs(want), "scoped bob");
}
/// The engine counts each index's filtered queries by key.
#[tokio::test]
async fn queries_are_counted_by_filter_key() {
let fixture = Fixture::new().await;
fixture
.put((0..50).map(|i| row(i, author(i))).collect())
.await;
for (_, filter, _) in &FILTERS[..3] {
fixture
.query(&fixture.server, 0, filter, 5, &vector_of(1))
.await;
}
let year = FILTERS[13].1;
fixture
.query(&fixture.server, 0, year, 5, &vector_of(1))
.await;
let stats = fixture
.server
.filter_stats(Request::new(pb::FilterStatsRequest {
table: Some(fixture.table.clone()),
}))
.await
.unwrap()
.into_inner();
let author = stats.keys.iter().find(|k| k.key == "author").unwrap();
assert_eq!(author.queries, 4);
assert!(author.string_operand && !author.number_operand);
let year = stats.keys.iter().find(|k| k.key == "year").unwrap();
assert_eq!(year.queries, 1);
assert!(year.number_operand && !year.string_operand);
}
/// A PutVectors that opened the table before its key column was added and
/// commits after: Lance rebases the write and its rows hold NULL in the new
/// column, which the write then fills from the rows' metadata.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_write_that_raced_the_column_add_fills_the_column() {
let fixture = Fixture::new().await;
fixture
.put((0..300).map(|i| row(i, author(i))).collect())
.await;
let unhealed = fixture.dataset(0).await;
let healed = fixture.dataset(0).await;
fixture.index_author().await;
let nulls = |dataset: Dataset| async move {
dataset
.count_rows(Some(format!("{COLUMN} IS NULL")))
.await
.unwrap()
};
let rows: Vec<pb::Vector> = (250..400).map(|i| row(i, author(i))).collect();
let written = crate::service::merge_vectors(unhealed, &rows)
.await
.unwrap();
assert!(written.schema().field(COLUMN).is_some());
assert!(nulls(written).await >= 100, "the race is reproduced");
let written = crate::service::write_vectors(healed, &rows).await.unwrap();
assert_eq!(nulls(written.clone()).await, 0);
let filter = FILTERS[1].1;
let latest = written.version().version;
let v = vector_of(5);
assert_same(
&fixture
.query(&fixture.server, latest, filter, 400, &v)
.await,
&fixture.flat(latest, filter, 400, &v).await,
"bob after the race",
);
assert_eq!(fixture.flat(latest, filter, 400, &v).await.len(), 4);
}
/// Two writes updating the same keys raced the column add, and the first
/// one's fill commits last: each write's rows get the values of its own
/// metadata, not the ones derived before it or from the other write.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_write_that_raced_the_column_add_derives_its_updated_rows() {
let fixture = Fixture::new().await;
fixture
.put((0..300).map(|i| row(i, author(i))).collect())
.await;
let first = fixture.dataset(0).await;
let second = fixture.dataset(0).await;
let written: Schema = first.schema().into();
fixture.index_author().await;
let rows = |name: &str| -> Vec<pb::Vector> {
(250..300)
.map(|i| row(i, Some(serde_json::json!(name))))
.collect()
};
let keys: Vec<String> = (250..300).map(|i| format!("k{i}")).collect();
let first = crate::service::merge_vectors(first, &rows("alice"))
.await
.unwrap();
let second = crate::service::merge_vectors(second, &rows("bob"))
.await
.unwrap();
crate::service::heal_columns(second, &written, &keys)
.await
.unwrap();
crate::service::heal_columns(first, &written, &keys)
.await
.unwrap();
let latest = fixture.dataset(0).await;
let count = |filter: String| {
let latest = latest.clone();
async move { latest.count_rows(Some(filter)).await.unwrap() }
};
assert_eq!(count(format!("{COLUMN} IS NULL")).await, 0);
for (name, filter) in [("alice", FILTERS[0].1), ("bob", FILTERS[1].1)] {
assert_eq!(
count(format!("array_has({COLUMN}, '{name}')")).await,
count(filter.to_string()).await,
"{name}"
);
}
let mut got = fixture
.server
.get_vectors(Request::new(pb::GetVectorsRequest {
table: Some(fixture.table.clone()),
keys: keys.clone(),
return_data: true,
return_metadata: true,
}))
.await
.unwrap()
.into_inner()
.vectors;
got.sort_by_key(|v| v.key[1..].parse::<usize>().unwrap());
assert_eq!(got, rows("bob"));
}
-7
View File
@@ -1,12 +1,5 @@
//! The VectorService gRPC server the S3 gateway talks to.
// PutVectors awaits two Lance merge_insert futures; proving the handler
// future Send takes deeper trait recursion than the default.
#![recursion_limit = "256"]
pub mod filter_stats;
#[cfg(test)]
mod indexed_keys_tests;
pub mod metrics;
pub mod observed;
pub mod read_cache;
-6
View File
@@ -52,11 +52,6 @@ struct Args {
/// tenth of the index's partitions, at least 8; more under a filter.
#[arg(long, env = "WEED_VECTOR_NPROBES")]
nprobes: Option<std::num::NonZeroUsize>,
/// A filtered query whose indexed metadata keys match at most this many
/// rows scores them exactly instead of searching the vector index; 0
/// never does.
#[arg(long, env = "WEED_VECTOR_EXACT_MATCH_LIMIT", default_value_t = weed_vector_core::query::DEFAULT_EXACT_MATCH_LIMIT)]
exact_match_limit: usize,
/// The cluster's security.toml: [grpc.vector] for this server,
/// [grpc.client] for the masters and filers, https.client for volume
/// servers under HTTPS, jwt.signing.read.key for signed volume reads,
@@ -175,7 +170,6 @@ async fn main() -> Result<()> {
.then(|| ReadCache::new(args.read_cache_mb << 20).with_metrics(metrics.clone()));
let mut service = VectorServer::new()
.with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get))
.with_exact_match_limit(Some(args.exact_match_limit))
.with_lance_caches(args.index_cache_mb << 20, args.metadata_cache_mb << 20)
.with_storage(reads);
if let Some(cache) = &cache {
@@ -10,10 +10,6 @@ use tonic_health::{ServingStatus, server::HealthReporter};
use crate::read_cache::ReadCache;
/// The gRPC health service an engine reports serving when its writes derive
/// the indexed metadata key columns.
pub const METADATA_COLUMNS_CAPABILITY: &str = "weed.vector.capability.metadata_columns";
#[derive(Clone)]
pub struct Metrics {
pub registry: Registry,
@@ -124,11 +120,6 @@ impl Metrics {
health
.set_service_status("vector_pb.VectorService", status)
.await;
// A capability, not readiness: the admin refuses to index a
// metadata key while an engine without it could write the table.
health
.set_service_status(METADATA_COLUMNS_CAPABILITY, ServingStatus::Serving)
.await;
}
}
@@ -78,15 +78,6 @@ impl VectorService for ObservedService {
}
result
}
async fn filter_stats(
&self,
request: Request<pb::FilterStatsRequest>,
) -> Result<Response<pb::FilterStatsResponse>, Status> {
let mut in_flight = self.metrics.start("FilterStats");
let result = self.inner.filter_stats(request).await;
in_flight.complete(result.as_ref().err().map(|e| e.code()));
result
}
}
#[cfg(test)]
+42 -182
View File
@@ -19,9 +19,7 @@ use weed_lance_worker::catalog::TableDescription;
use weed_lance_worker::dataset;
use weed_lance_worker::direct_read::{DirectReads, Wrappers};
use weed_lance_worker::jobs::vector_index::is_vector_table;
use weed_lance_worker::metadata_index;
use weed_vector_core::index;
use weed_vector_core::indexed_filter;
use weed_vector_core::maintenance::SharedLeases;
use weed_vector_core::query::{self, Query};
use weed_vector_core::table::VectorTable;
@@ -169,13 +167,14 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
};
let vectors_array =
FixedSizeListArray::try_new(item_field.clone(), dimension as i32, values, None)?;
let json: Vec<Option<serde_json::Value>> = vectors
let metas = vectors
.iter()
.map(|v| v.metadata.as_ref().map(struct_to_json))
.collect();
let metas = json
.iter()
.map(|m| m.as_ref().map(serde_json::to_string).transpose())
.map(|v| {
v.metadata
.as_ref()
.map(|m| serde_json::to_string(&struct_to_json(m)))
.transpose()
})
.collect::<Result<Vec<_>, _>>()?;
// The stored column is lance's jsonb (LargeBinary) once the field carries
// the arrow.json extension; encode the strings, else keep the Utf8 batch.
@@ -188,144 +187,11 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
}
_ => Arc::new(StringArray::from(metas)),
};
let (keys, vectors_array): (ArrayRef, ArrayRef) = (Arc::new(keys), Arc::new(vectors_array));
// An indexed metadata key's column is derived from the row's metadata,
// whatever the key's state.
let columns = schema
.fields()
.iter()
.map(|field| match field.name().as_str() {
KEY_COLUMN => Ok(keys.clone()),
VECTOR_COLUMN => Ok(vectors_array.clone()),
METADATA_COLUMN => Ok(metadata_array.clone()),
name => match metadata_index::indexed_key(field) {
Some(indexed) => Ok(metadata_index::column_from_json(
json.iter().map(Option::as_ref),
&indexed.key,
)),
None => bail!("the table has a column {name} that PutVectors cannot fill"),
},
})
.collect::<Result<Vec<_>>>()?;
RecordBatch::try_new(schema, columns).map_err(Into::into)
}
/// Upserts `vectors` by key into `dataset`, the version the request opened,
/// then fills the indexed-key columns a concurrent column add left empty.
pub(crate) async fn write_vectors(
dataset: Dataset,
vectors: &[pb::Vector],
) -> Result<Dataset, Status> {
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
let dataset = merge_vectors(dataset, vectors).await?;
let keys: Vec<String> = vectors.iter().map(|v| v.key.clone()).collect();
heal_columns(dataset, &written, &keys)
.await
.map_err(internal)
}
/// The upsert itself, in the schema of the version `dataset` is at.
pub(crate) async fn merge_vectors(
dataset: Dataset,
vectors: &[pb::Vector],
) -> Result<Dataset, Status> {
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
.map_err(invalid)?
.dimension;
let batch = batch_of(
vectors,
dimension,
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
RecordBatch::try_new(
schema,
vec![Arc::new(keys), Arc::new(vectors_array), metadata_array],
)
.map_err(invalid)?;
// Upsert by key in one merge commit: no window where the old rows are
// already deleted, and a losing writer fails the request instead of
// leaving duplicate keys.
let mut builder = MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
.map_err(internal)?;
builder.when_matched(WhenMatched::UpdateAll);
builder.when_not_matched(WhenNotMatched::InsertAll);
let (dataset, _stats) = builder
.try_build()
.map_err(internal)?
.execute_batches(vec![batch])
.await
.map_err(internal)?;
Ok(Arc::unwrap_or_clone(dataset))
}
/// Attempts [`heal_columns`] makes against writes committing meanwhile.
const HEAL_ATTEMPTS: usize = 10;
/// Lance rebases a write onto a column add committed after the write opened
/// its version: the rows it inserted hold NULL in the new column and the
/// rows it updated keep the value derived from their old metadata. Derive
/// such columns again for the rows of `keys` from their stored metadata.
pub(crate) async fn heal_columns(
mut dataset: Dataset,
written: &Schema,
keys: &[String],
) -> Result<Dataset> {
for _ in 0..HEAL_ATTEMPTS {
let schema: Schema = dataset.schema().into();
let missing: Vec<metadata_index::IndexedKey> = metadata_index::indexed_keys(&schema)
.into_iter()
.filter(|k| written.field_with_name(&k.column).is_err())
.collect();
if missing.is_empty() || keys.is_empty() {
return Ok(dataset);
}
let mut scanner = dataset.scan();
scanner
.filter(&key_list(keys))?
.project(&[KEY_COLUMN, METADATA_COLUMN])?;
let rows: Vec<RecordBatch> = scanner.try_into_stream().await?.try_collect().await?;
let mut fields = vec![schema.field_with_name(KEY_COLUMN)?.clone()];
for k in &missing {
fields.push(schema.field_with_name(&k.column)?.clone());
}
let patch_schema = Arc::new(Schema::new(fields));
let mut patches = Vec::new();
for batch in rows.iter().filter(|b| b.num_rows() > 0) {
let (Some(keys), Some(metadata)) = (
batch.column_by_name(KEY_COLUMN),
batch.column_by_name(METADATA_COLUMN),
) else {
bail!("the scan did not return keys and metadata");
};
let mut columns = vec![keys.clone()];
for k in &missing {
columns.push(metadata_index::column_from_metadata(
metadata.as_ref(),
&k.key,
)?);
}
patches.push(RecordBatch::try_new(patch_schema.clone(), columns)?);
}
if patches.is_empty() {
return Ok(dataset);
}
// Lance's own retry would join these values onto rows a later write
// replaced; derive them again from the version that write left.
let mut builder =
MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec![KEY_COLUMN.to_string()])?;
builder
.when_matched(WhenMatched::UpdateAll)
.when_not_matched(WhenNotMatched::DoNothing)
.conflict_retries(0);
match builder.try_build()?.execute_batches(patches).await {
Ok((healed, _stats)) => {
tracing::info!(
rows = rows.iter().map(|b| b.num_rows()).sum::<usize>(),
"derived indexed metadata key columns a concurrent column add left stale"
);
return Ok(Arc::unwrap_or_clone(healed));
}
Err(lance::Error::TooMuchWriteContention { .. }) => dataset.checkout_latest().await?,
Err(e) => return Err(e.into()),
}
}
bail!("writes kept replacing the rows while their indexed key columns were derived")
.map_err(Into::into)
}
fn struct_to_json(s: &Struct) -> serde_json::Value {
@@ -364,7 +230,7 @@ fn json_to_value(v: serde_json::Value) -> Value {
Value { kind: Some(kind) }
}
pub(crate) fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
Struct {
fields: map
.into_iter()
@@ -662,8 +528,6 @@ pub struct VectorServer {
lance_cache_bytes: (usize, usize),
leases: SharedLeases,
nprobes: Option<usize>,
exact_match_limit: Option<usize>,
filter_stats: crate::filter_stats::FilterStats,
/// URI schemes a table's location may have.
schemes: &'static [&'static str],
}
@@ -687,8 +551,6 @@ impl VectorServer {
handles: moka::sync::Cache::new(MAX_HANDLES),
leases: SharedLeases::default(),
nprobes: None,
exact_match_limit: None,
filter_stats: Default::default(),
schemes: &["s3"],
}
}
@@ -706,13 +568,6 @@ impl VectorServer {
self
}
/// Matches of a filter's indexed metadata keys scored exactly instead of
/// searched through the vector index, at most; None keeps the default.
pub fn with_exact_match_limit(mut self, limit: Option<usize>) -> Self {
self.exact_match_limit = limit;
self
}
/// Keep tables on the filers and volume servers, not through the gateway.
/// Unset only for tests on an in-process store.
pub fn with_storage(mut self, storage: DirectReads) -> Self {
@@ -953,7 +808,29 @@ impl VectorService for VectorServer {
version: dataset.version().version,
}));
}
let dataset = write_vectors(dataset, &req.vectors).await?;
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
.map_err(invalid)?
.dimension;
let batch = batch_of(
&req.vectors,
dimension,
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
)
.map_err(invalid)?;
// Upsert by key in one merge commit: no window where the old rows are
// already deleted, and a losing writer fails the request instead of
// leaving duplicate keys.
let mut builder =
MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
.map_err(internal)?;
builder.when_matched(WhenMatched::UpdateAll);
builder.when_not_matched(WhenNotMatched::InsertAll);
let (dataset, _stats) = builder
.try_build()
.map_err(internal)?
.execute_batches(vec![batch])
.await
.map_err(internal)?;
Ok(Response::new(pb::PutVectorsResponse {
version: dataset.version().version,
}))
@@ -1031,9 +908,14 @@ impl VectorService for VectorServer {
0 => DEFAULT_PAGE,
n => n.min(MAX_PAGE),
};
// A purge keeps only the latest version, so lease the one this page reads.
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
if segments > 1 {
// Data and metadata come from a second read of this version, so
// lease it as get_vectors does.
let _lease = if columns.len() > 1 {
self.leases.try_pin(&dataset).await.map_err(internal)?
} else {
None
};
let (vectors, next) = list_segment(
&dataset,
&columns,
@@ -1071,7 +953,6 @@ impl VectorService for VectorServer {
&self,
request: Request<pb::QueryVectorsRequest>,
) -> Result<Response<pb::QueryVectorsResponse>, Status> {
let started = std::time::Instant::now();
let req = request.into_inner();
let table = req
.table
@@ -1081,11 +962,6 @@ impl VectorService for VectorServer {
// while this query runs against it.
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
let resolved = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN)).map_err(invalid)?;
let filter_keys = if req.filter.is_empty() {
Default::default()
} else {
indexed_filter::filter_keys(&dataset, &req.filter).unwrap_or_default()
};
if req.query_vector.len() != resolved.dimension {
return Err(Status::invalid_argument(format!(
"query vector carries {} floats, the index dimension is {}",
@@ -1110,7 +986,6 @@ impl VectorService for VectorServer {
nprobes: self.nprobes,
index_segments: None,
fragments: None,
exact_match_limit: self.exact_match_limit,
};
let hits = query::ann(&dataset, &resolved, &query)
.await
@@ -1186,23 +1061,8 @@ impl VectorService for VectorServer {
data: data_map.remove(&hit.key).unwrap_or_default(),
});
}
self.filter_stats
.record(&table.id.join("/"), &filter_keys, started.elapsed());
Ok(Response::new(pb::QueryVectorsResponse { vectors }))
}
async fn filter_stats(
&self,
request: Request<pb::FilterStatsRequest>,
) -> Result<Response<pb::FilterStatsResponse>, Status> {
let table = request
.into_inner()
.table
.ok_or_else(|| Status::invalid_argument("missing table"))?;
Ok(Response::new(
self.filter_stats.snapshot(&table.id.join("/")),
))
}
}
#[cfg(test)]
+2 -2
View File
@@ -3195,9 +3195,9 @@ dependencies = [
[[package]]
name = "redb"
version = "4.6.0"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "096b1dd5ec053c9c2a5a26a93aafeadf73ebcb710ea54dc3151f17a8e20535ef"
checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36"
dependencies = [
"libc",
]
+3
View File
@@ -19,6 +19,9 @@ path = "src/main.rs"
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
default = ["5bytes"]
5bytes = []
# Unstable redb cursor bulk-load for full_rebuild. Off in production.
# Pulls redb's experimental_cursor (and therefore experimental-api-5).
redb-experimental-cursor = ["redb/experimental_cursor"]
[lints.clippy]
# Protobuf message literals keep `..Default::default()` on purpose: it is
-1
View File
@@ -147,7 +147,6 @@ service VolumeServer {
message BatchDeleteRequest {
repeated string file_ids = 1;
bool skip_cookie_check = 2;
bool fsync = 3;
}
message BatchDeleteResponse {
+1 -2
View File
@@ -138,8 +138,7 @@ pub struct Cli {
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
pub concurrent_upload_limit_mb: u32,
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
/// the limit are sent to a replica unless readMode is local.
/// Limit total concurrent download size in MB, 0 means unlimited.
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
pub concurrent_download_limit_mb: u32,
-2
View File
@@ -416,7 +416,6 @@ async fn run(
.to_string_lossy()
.into_owned()
},
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight,
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
@@ -451,7 +450,6 @@ async fn run(
{
let mut store = state.store.write().unwrap();
store.store_dirs = config.folders.clone();
for (i, dir) in config.folders.iter().enumerate() {
let idx_dir = if config.idx_folder.is_empty() {
dir.as_str()
-14
View File
@@ -85,19 +85,6 @@ pub static VACUUMING_COMMIT_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(||
.expect("metric can be created")
});
/// Live needles vacuum dropped because their data could not be read back
/// intact, with label `collection`.
pub static VACUUMING_DROPPED_NEEDLES_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_vacuuming_dropped_needles",
"Counter of live needles dropped by vacuum because their data could not be read back intact.",
),
&["collection"],
)
.expect("metric can be created")
});
/// Vacuuming duration histogram with label `type` (compact/commit).
pub static VACUUMING_HISTOGRAM: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
@@ -415,7 +402,6 @@ pub fn register_metrics() {
Box::new(VACUUMING_COMPACT_COUNTER.clone()),
Box::new(VACUUMING_COMMIT_COUNTER.clone()),
Box::new(VACUUMING_HISTOGRAM.clone()),
Box::new(VACUUMING_DROPPED_NEEDLES_COUNTER.clone()),
Box::new(VOLUME_GAUGE.clone()),
Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
Box::new(MAX_VOLUMES.clone()),
-80
View File
@@ -106,39 +106,6 @@ pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims,
Ok(data.claims)
}
/// Reports a signed, fid-bound write token whose only failure is expiry. It
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
pub fn is_expired_volume_jwt(signing_key: &SigningKey, token: &str, expected_fid: &str) -> bool {
match decode_jwt(signing_key, token) {
Err(JwtError::Jwt(e))
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
_ => return false,
}
#[derive(Deserialize)]
struct ExpiredClaims {
fid: Option<String>,
iat: Option<u64>,
}
let mut validation = Validation::new(Algorithm::HS256);
validation.required_spec_claims.clear();
validation.validate_exp = false;
validation.validate_nbf = true;
validation.leeway = 0;
let Ok(data) = decode::<ExpiredClaims>(
token,
&DecodingKey::from_secret(&signing_key.0),
&validation,
) else {
return false;
};
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
}
// ============================================================================
// Guard
// ============================================================================
@@ -443,53 +410,6 @@ mod tests {
assert!(decode_jwt(&key2, &token).is_err());
}
#[test]
fn test_is_expired_volume_jwt_classifies_expiry_only() {
let key = SigningKey::from_string("write-key");
let fid = "3,01637037d6";
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let sign = |claims: serde_json::Value, key: &SigningKey| {
encode(
&Header::new(Algorithm::HS256),
&claims,
&EncodingKey::from_secret(&key.0),
)
.unwrap()
};
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
assert!(is_expired_volume_jwt(&key, &expired, fid));
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7"));
assert!(!is_expired_volume_jwt(
&SigningKey::from_string("other-key"),
&expired,
fid
));
assert!(!is_expired_volume_jwt(&SigningKey(vec![]), &expired, fid));
let immature = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &immature, fid));
let future_issued = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &future_issued, fid));
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
assert!(!is_expired_volume_jwt(&key, &live, fid));
let unsigned = format!(
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
base64::Engine::encode(
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
)
);
assert!(!is_expired_volume_jwt(&key, &unsigned, fid));
}
#[test]
fn test_guard_empty_whitelist() {
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
File diff suppressed because it is too large. Load diff
+43 -453
View File
@@ -165,10 +165,8 @@ impl SourceReadRequest {
fn served_from_source(&self, n: &Needle) -> bool {
if self.is_head {
// HEAD is meta-only unless the payload is needed to expand a
// manifest or to answer a decoded or transformed length.
return !self.has_image_ops
&& !n.is_compressed()
&& !(n.is_chunk_manifest() && !self.bypass_cm);
// manifest or to answer an image transform's length.
return !self.has_image_ops && !(n.is_chunk_manifest() && !self.bypass_cm);
}
self.direct(n) && (self.has_range || n.data_size > STREAMING_THRESHOLD)
}
@@ -485,18 +483,6 @@ fn extract_file_id(path: &str) -> String {
format!("{},{}", parsed.vid, fid)
}
fn is_multipart_form_data(headers: &HeaderMap) -> bool {
headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.split(';').next())
.is_some_and(|media_type| {
media_type
.trim()
.eq_ignore_ascii_case("multipart/form-data")
})
}
fn streaming_chunk_size(read_buffer_size_bytes: usize, data_size: usize) -> usize {
std::cmp::min(
read_buffer_size_bytes.max(DEFAULT_STREAMING_CHUNK_SIZE),
@@ -685,7 +671,6 @@ async fn batch_delete_file_ids(
.batch_delete(volume_server_pb::BatchDeleteRequest {
file_ids: batch.clone(),
skip_cookie_check: true,
fsync: false,
})
.await
.map_err(|e| format!("batch delete {}: {}", grpc_addr, e))?
@@ -861,12 +846,7 @@ async fn do_replicated_request(
futures.push(async move {
match req_builder.send().await {
Ok(r)
if r.status().is_success()
|| (allow_delete && r.status() == StatusCode::NOT_FOUND) =>
{
Ok(())
}
Ok(r) if r.status().is_success() => Ok(()),
Ok(r) => {
crate::metrics::UPLOAD_ERROR_COUNTER
.with_label_values(&[&r.status().as_u16().to_string()])
@@ -933,6 +913,7 @@ async fn proxy_or_redirect_to_target(
state: &VolumeServerState,
info: ProxyRequestInfo,
vid: VolumeId,
allow_local_redirect: bool,
) -> Response {
// Look up volume locations from master
let locations = match lookup_volume(
@@ -980,7 +961,10 @@ async fn proxy_or_redirect_to_target(
match state.read_mode {
ReadMode::Proxy => proxy_request(state, &info, target).await,
ReadMode::Redirect => redirect_request(&info, target, &state.outgoing_http_scheme),
ReadMode::Local => StatusCode::NOT_FOUND.into_response(),
ReadMode::Local if allow_local_redirect => {
redirect_request(&info, target, &state.outgoing_http_scheme)
}
ReadMode::Local => unreachable!(),
}
}
@@ -1349,7 +1333,6 @@ async fn get_or_head_handler_inner(
&query,
&ext,
request_kind.has_image_ops,
request_kind.has_range,
&mut response_headers,
) {
ControlFlow::Continue(data) => data,
@@ -1448,7 +1431,7 @@ async fn proxy_missing_volume(
None => return StatusCode::NOT_FOUND.into_response(),
};
proxy_or_redirect_to_target(state, info, vid).await
proxy_or_redirect_to_target(state, info, vid, false).await
}
/// Go's `proxied` query param check (loop prevention).
@@ -1481,10 +1464,8 @@ async fn check_download_limit(
// Go tries proxy to replica once before the wait loop
// (checkDownloadLimit); it does not retry on each wakeup.
let should_try_replica = state.read_mode != ReadMode::Local
&& !is_proxied_query(&query_string)
&& !state.current_master().await.is_empty()
&& {
let should_try_replica =
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
let store = state.store.read().unwrap();
store
.find_volume(vid)
@@ -1494,24 +1475,25 @@ async fn check_download_limit(
&& let Some(info) =
build_proxy_request_info(path, request_headers, &query_string, method)
{
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid).await);
return ControlFlow::Break(proxy_or_redirect_to_target(state, info, vid, true).await);
}
// Blocking wait loop (Go's waitForDownloadSlot)
loop {
let notified = state.download_notify.notified();
if state.inflight_download_bytes.load(Ordering::Relaxed)
<= state.concurrent_download_limit
if tokio::time::timeout_at(deadline, state.download_notify.notified())
.await
.is_err()
{
break;
}
if tokio::time::timeout_at(deadline, notified).await.is_err() {
return ControlFlow::Break(json_error_with_query(
StatusCode::TOO_MANY_REQUESTS,
"download limit exceeded",
uri.query(),
));
}
let current = state.inflight_download_bytes.load(Ordering::Relaxed);
if current <= state.concurrent_download_limit {
break;
}
}
}
ControlFlow::Continue(true)
@@ -1651,8 +1633,8 @@ async fn read_volume_needle(
let can_stream =
can_direct_source_read && n.data_size > STREAMING_THRESHOLD && !has_range && !is_head;
// HEAD is meta-only when served_from_source loaded no payload. A compressed
// needle, or a chunk manifest without cm=false, is read in full.
// HEAD is meta-only when served_from_source loaded no payload. A chunk
// manifest without cm=false is read in full so expansion can run.
let can_handle_head_from_meta = stream_info.is_some() && is_head;
let can_handle_range_from_source = can_direct_source_read && has_range;
@@ -2047,7 +2029,6 @@ fn buffered_payload(
query: &ReadQueryParams,
ext: &str,
needs_image_ops: bool,
has_range: bool,
response_headers: &mut HeaderMap,
) -> ControlFlow<Response, Vec<u8>> {
// Handle compressed data: if needle is compressed, either pass through or decompress
@@ -2076,9 +2057,7 @@ fn buffered_payload(
.get(header::ACCEPT_ENCODING)
.and_then(|v| v.to_str().ok())
.unwrap_or("");
// A range is over the decoded bytes, so only a whole read passes gzip through.
if !has_range
&& accept_encoding.contains("gzip")
if accept_encoding.contains("gzip")
&& data.len() >= 2
&& data[0] == 0x1f
&& data[1] == 0x8b
@@ -2801,28 +2780,14 @@ pub async fn post_handler(
// JWT check for writes
let file_id = extract_file_id(&path);
let token = extract_jwt(&headers, request.uri());
let expired_jwt = {
let guard = state.guard.read().unwrap();
match guard.check_jwt_for_file(token.as_deref(), &file_id, true) {
Ok(()) => None,
Err(_) => Some(
is_multipart_form_data(&headers)
&& token.as_deref().is_some_and(|token| {
crate::security::is_expired_volume_jwt(&guard.signing_key, token, &file_id)
}),
),
}
};
if let Some(expired_jwt) = expired_jwt {
let mut response =
json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
if expired_jwt {
response.headers_mut().insert(
"x-seaweed-write-outcome",
header::HeaderValue::from_static("expired-jwt-before-append"),
);
}
return response;
if state
.guard
.read()
.unwrap()
.check_jwt_for_file(token.as_deref(), &file_id, true)
.is_err()
{
return json_error_with_query(StatusCode::UNAUTHORIZED, "wrong jwt", Some(&query));
}
// Upload throttling: check inflight bytes against limit
@@ -3692,30 +3657,22 @@ pub async fn delete_handler(
// Read needle first to validate cookie (matching Go behavior)
let original_cookie = cookie;
let del_fields =
serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query).unwrap_or_default();
let del_field = |key: &str| {
del_fields
.iter()
.find(|(name, _)| name == key)
.map(|(_, value)| value.as_str())
};
let is_replicate = del_field("type") == Some("replicate");
let fsync = del_field("fsync") == Some("true");
let is_replicate = serde_urlencoded::from_str::<Vec<(String, String)>>(&del_query)
.unwrap_or_default()
.into_iter()
.find(|(name, _)| name == "type")
.is_some_and(|(_, value)| value == "replicate");
{
let vol = {
let store = state.store.read().unwrap();
store.find_volume(vid).map(|(_, vol)| vol)
};
let replicated = vol
.as_ref()
.is_some_and(|v| v.super_block().replica_placement.get_copy_count() > 1);
let mut found = vol
.as_ref()
.ok_or(crate::storage::volume::VolumeError::NotFound)
.and_then(|vol| vol.read_needle_with_option(&mut n, &mut delete_read_option()));
if found.is_err()
&& (is_replicate || replicated)
&& is_replicate
&& let Some(vol) = &vol
{
// the write this delete undoes can still be queued here
@@ -3724,42 +3681,7 @@ pub async fn delete_handler(
}
match found {
Ok(_) => {}
Err(e) => {
// 404 tells the forwarding replica the needle is gone here, so a
// read that failed for any other reason must not answer it
if !matches!(
e,
crate::storage::volume::VolumeError::NotFound
| crate::storage::volume::VolumeError::Deleted
) {
return json_error_with_query(
StatusCode::INTERNAL_SERVER_ERROR,
format!("Deletion Failed: {}", e),
Some(&del_query),
);
}
// a retry of a delete whose replica fan-out failed finds nothing
// here, but the other replicas may still hold the needle
if !is_replicate
&& replicated
&& let Err(e) = do_replicated_request(
&state,
vid.0,
Method::DELETE,
&path,
&del_query,
&headers,
None,
)
.await
{
tracing::error!("replicated delete failed: {}", e);
return json_error_with_query(
StatusCode::INTERNAL_SERVER_ERROR,
format!("replication failed: {}", e),
Some(&del_query),
);
}
Err(_) => {
let result = DeleteResult { size: 0 };
return json_response_with_params(
StatusCode::NOT_FOUND,
@@ -3841,16 +3763,7 @@ pub async fn delete_handler(
Ok(vol) if vol.is_no_write_or_delete() => {
Err(crate::storage::volume::VolumeError::ReadOnly(vid))
}
Ok(vol) if fsync => {
tokio::task::spawn_blocking(move || vol.delete_needle(&mut n, true))
.await
.unwrap_or_else(|e| {
Err(crate::storage::volume::VolumeError::Io(
std::io::Error::other(e),
))
})
}
Ok(vol) => vol.delete_needle(&mut n, false),
Ok(vol) => vol.delete_needle(&mut n),
}
};
@@ -5059,13 +4972,13 @@ fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
}
fn maybe_decompress_gzip(data: &[u8]) -> Result<Vec<u8>, GunzipError> {
use flate2::read::MultiGzDecoder;
use flate2::read::GzDecoder;
use std::io::Read;
// Cap the output so a crafted, highly-compressible (gzip-bomb) needle cannot
// OOM the server when decompressed on read or upload. take(limit+1) lets us
// tell "exactly at the limit" apart from "over it", which we reject as
// TooLarge so callers fail the request instead of silently using raw bytes.
let mut decoder = MultiGzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
let mut decoder = GzDecoder::new(data).take(MAX_EXPANSION_BYTES + 1);
let mut decompressed = Vec::new();
decoder
.read_to_end(&mut decompressed)
@@ -5154,34 +5067,6 @@ mod tests {
(addr, receipts)
}
/// A replica that no longer holds the needle answers a forwarded delete
/// with 404, which means it is gone there too.
#[tokio::test]
async fn test_replicated_delete_takes_a_replica_404_as_done() {
let tmp = tempfile::TempDir::new().unwrap();
let app = axum::Router::new().fallback(|| async { StatusCode::NOT_FOUND });
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let peer = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let state = remote_manifest_state(&tmp, spawn_lookup(peer).await);
for (method, done) in [
(axum::http::Method::DELETE, true),
(axum::http::Method::POST, false),
] {
let sent = do_replicated_request(
&state,
1,
method.clone(),
"/1,01ab",
"",
&axum::http::HeaderMap::new(),
None,
)
.await;
assert_eq!(sent.is_ok(), done, "{method}: {sent:?}");
}
}
/// The whole replicated write path, not only the request builder: the
/// replica receives the forward, never answers, and the write fails at
/// the bound for its body with an error naming that replica.
@@ -5583,83 +5468,6 @@ mod tests {
assert_eq!(extract_jwt(&headers, &uri), None);
}
#[tokio::test]
async fn http_expired_write_jwt_is_marked_before_append() {
use crate::security::SigningKey;
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
use tower::ServiceExt;
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let key = SigningKey(b"write-secret".to_vec());
state.guard.write().unwrap().signing_key = key.clone();
let volume = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
let before = volume.dat_file_size().unwrap();
let fid = format!("1,{:x}{TEST_COOKIE:08x}", 0x6e7a_0c09u64);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let sign = |key: &[u8]| {
encode(
&Header::new(Algorithm::HS256),
&serde_json::json!({"fid": fid, "exp": now - 10}),
&EncodingKey::from_secret(key),
)
.unwrap()
};
let boundary = "expiredjwtboundary";
let body = format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a\"\r\n\r\nexpired-bytes\r\n--{boundary}--\r\n"
);
for (token, content_type, marked) in [
(
sign(&key.0),
format!("multipart/form-data; boundary={boundary}"),
true,
),
(
sign(b"other-secret"),
format!("multipart/form-data; boundary={boundary}"),
false,
),
(sign(&key.0), "application/octet-stream".to_string(), false),
] {
let response = super::super::volume_server::build_admin_router(state.clone())
.oneshot(
Request::builder()
.method(Method::POST)
.uri(format!("/{fid}"))
.header(header::AUTHORIZATION, format!("Bearer {token}"))
.header(header::CONTENT_TYPE, &content_type)
.body(Body::from(body.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::UNAUTHORIZED,
"{content_type}"
);
assert_eq!(
response
.headers()
.get("x-seaweed-write-outcome")
.map(|value| value.to_str().unwrap()),
marked.then_some("expired-jwt-before-append"),
"{content_type}"
);
}
assert_eq!(volume.dat_file_size().unwrap(), before);
}
#[test]
fn test_handle_range_single() {
let data = b"hello world";
@@ -6473,7 +6281,6 @@ mod tests {
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
@@ -6555,88 +6362,6 @@ mod tests {
test_state_with_store(store)
}
#[tokio::test]
async fn download_limit_local_read_mode_waits_instead_of_proxying() {
use crate::storage::super_block::ReplicaPlacement;
let tmp = tempfile::TempDir::new().unwrap();
let mut state = volume_test_state(&tmp);
{
let s = Arc::get_mut(&mut state).unwrap();
s.read_mode = ReadMode::Local;
s.concurrent_download_limit = 1;
s.inflight_download_data_timeout = std::time::Duration::from_millis(50);
}
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
state.inflight_download_bytes.store(2, Ordering::Relaxed);
state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
.unwrap();
let path = "/1,0123456789abcdef";
let uri: axum::http::Uri = path.parse().unwrap();
let flow = check_download_limit(
&state,
&uri,
&HeaderMap::new(),
path,
VolumeId(1),
&Method::GET,
)
.await;
match flow {
ControlFlow::Break(resp) => assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS),
ControlFlow::Continue(_) => panic!("read over the download limit must not proceed"),
}
}
#[tokio::test]
async fn download_limit_wait_sees_release_before_waiting() {
let mut state = streaming_test_state();
{
let s = Arc::get_mut(&mut state).unwrap();
s.read_mode = ReadMode::Proxy;
s.concurrent_download_limit = 1;
s.inflight_download_data_timeout = std::time::Duration::from_secs(5);
}
*state.current_master_url.write().await = "127.0.0.1:1".to_string();
state.inflight_download_bytes.store(2, Ordering::Relaxed);
let master_guard = state.current_master_url.write().await;
let waiter = tokio::spawn({
let state = state.clone();
async move {
let path = "/1,0123456789abcdef";
let uri: axum::http::Uri = path.parse().unwrap();
check_download_limit(
&state,
&uri,
&HeaderMap::new(),
path,
VolumeId(1),
&Method::GET,
)
.await
}
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
state.inflight_download_bytes.store(0, Ordering::Relaxed);
state.download_notify.notify_waiters();
drop(master_guard);
let flow = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
.await
.expect("waiter missed the released download slot")
.unwrap();
assert!(matches!(flow, ControlFlow::Continue(true)));
}
/// Write a needle to volume 1 and return its URL path.
fn put_test_needle(state: &VolumeServerState, id: u64, data: &[u8]) -> String {
put_test_needle_with(state, id, data, |_| {})
@@ -7506,28 +7231,6 @@ mod tests {
}
}
/// A needle of concatenated gzip members decodes to all of them, as Go's
/// gzip reader does, so a range spans the whole decoded content.
#[tokio::test]
async fn test_get_range_over_multi_member_gzip_needle() {
const ID: u64 = 0x6e7a_0503;
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let mut gz = try_gzip_data(b"hello").unwrap();
gz.extend(try_gzip_data(b"world").unwrap());
let path = put_test_needle_with(&state, ID, &gz, |n| n.set_is_compressed());
let (status, headers, body) =
send_read(&state, Method::GET, &path, Some(b"bytes=3-7")).await;
assert_eq!(status, StatusCode::PARTIAL_CONTENT);
assert_eq!(body, b"lowor");
assert_eq!(headers["Content-Range"], "bytes 3-7/10");
let (status, _, body) = send_read(&state, Method::GET, &path, None).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body, b"helloworld");
}
/// An EC needle is served from memory: its GET parses Range like any
/// other, and its HEAD ignores it as Go's writeResponseContent does.
#[tokio::test]
@@ -8874,7 +8577,7 @@ mod tests {
.store
.write()
.unwrap()
.delete_volume_needle(VolumeId(1), &mut filler, false)
.delete_volume_needle(VolumeId(1), &mut filler)
.unwrap();
let data_offset = || {
let mut plan = crate::storage::store::Store::needle_read_plan(
@@ -9065,119 +8768,6 @@ mod tests {
}
}
/// A delete whose fan-out failed was already applied here, so the
/// client's retry finds nothing locally; it must still reach the replica
/// that kept the needle, or the replicas stay different.
#[tokio::test]
async fn test_delete_retry_reaches_the_replica_after_a_failed_fan_out() {
use crate::storage::super_block::ReplicaPlacement;
use axum::response::IntoResponse;
use axum::{Json, Router, middleware, routing::get};
let replicated = |state: &Arc<VolumeServerState>| {
let vol = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
vol.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
.unwrap();
vol
};
let tmp = tempfile::TempDir::new().unwrap();
let mut state = volume_test_state(&tmp);
Arc::get_mut(&mut state).unwrap().self_url = "127.0.0.1:8080".to_string();
let volume = replicated(&state);
let peer_tmp = tempfile::TempDir::new().unwrap();
let peer = volume_test_state(&peer_tmp);
let peer_volume = replicated(&peer);
let id = 0x6e7a_0c06u64;
let path = put_test_needle(&state, id, b"replicated delete");
put_test_needle(&peer, id, b"replicated delete");
let live = |vol: &Arc<crate::storage::volume::Volume>| {
vol.read_needle(&mut Needle {
id: NeedleId(id),
..Needle::default()
})
.is_ok()
};
let fail_next = Arc::new(std::sync::atomic::AtomicBool::new(true));
let app = super::super::volume_server::build_admin_router(peer).layer(middleware::from_fn(
move |request: Request<Body>, next: middleware::Next| {
let fail = fail_next.swap(false, Ordering::SeqCst);
async move {
if fail {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
}
next.run(request).await
}
},
));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let peer_addr = listener.local_addr().unwrap();
let peer_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let master = Router::new().route(
"/dir/lookup",
get(move || async move {
Json(serde_json::json!({"locations": [
{"url": "127.0.0.1:8080"},
{"url": peer_addr.to_string()}
]}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
*state.current_master_url.write().await = listener.local_addr().unwrap().to_string();
let master_server =
tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
assert_eq!(
send_delete(&state, &path).await,
StatusCode::INTERNAL_SERVER_ERROR
);
assert!(!live(&volume));
assert!(live(&peer_volume));
assert_eq!(send_delete(&state, &path).await, StatusCode::NOT_FOUND);
assert!(!live(&peer_volume));
master_server.abort();
peer_server.abort();
}
/// A forwarding replica takes a 404 as the needle being gone here, so a
/// delete whose read fails for another reason answers 500.
#[tokio::test]
async fn test_delete_whose_read_fails_is_not_a_404() {
use std::io::{Seek, SeekFrom, Write};
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let id = 0x6e7a_0c07u64;
let path = put_test_needle(&state, id, b"unreadable");
let dat = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1
.dat_path();
let mut file = std::fs::OpenOptions::new().write(true).open(dat).unwrap();
// the needle's size field, right after the superblock, cookie and id
file.seek(SeekFrom::Start(8 + 4 + 8)).unwrap();
file.write_all(&u32::MAX.to_be_bytes()).unwrap();
file.sync_all().unwrap();
for query in ["", "?type=replicate"] {
assert_eq!(
send_delete(&state, &format!("{path}{query}")).await,
StatusCode::INTERNAL_SERVER_ERROR,
"{query}"
);
}
}
#[tokio::test]
async fn http_replication_preserves_query_authentication() {
use crate::security::{SigningKey, gen_jwt};
@@ -9651,8 +9241,8 @@ mod tests {
/// replicate delete of it finds nothing. The delete cancels the write, so
/// it is taken back when it lands instead of reviving the needle on this
/// copy alone, even when its handler was dropped while it waited in the
/// durable queue. A plain delete on an unreplicated volume that finds
/// nothing leaves the write alone.
/// durable queue. A plain delete that finds nothing is not sent on to the
/// replicas, so it leaves the write alone.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[allow(clippy::await_holding_lock)]
async fn test_replicate_delete_cancels_a_queued_write() {
-1
View File
@@ -1698,7 +1698,6 @@ mod tests {
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
+9 -31
View File
@@ -2384,45 +2384,23 @@ pub(crate) async fn recover_missing_ec_indexes(
"cannot look up peers to recover missing .ecx: {}",
e
);
Vec::new()
continue;
}
};
// Under the volume's lifecycle lock, so an in-flight VolumeEcShardsCopy
// of it, which may unlink the shards it renamed in, never has them
// mounted from under it.
let _lifecycle = state.ec_lifecycle_lock(m.vid).lock_owned().await;
let current = state
.store
.read()
.unwrap()
.collect_ec_volumes_missing_index_only(Some(m.vid))
.into_iter()
.find(|x| x.vid == m.vid && x.collection == m.collection);
// A copy holding the lock may have installed the index meanwhile: skip
// the fetch, but its shards still need the mount.
if let Some(current) = &current {
if peers.is_empty() {
tracing::warn!(
volume_id = m.vid.0,
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
);
continue;
}
if !fetch_ec_index_from_peers(state, current, &peers).await {
continue;
}
if peers.is_empty() {
tracing::warn!(
volume_id = m.vid.0,
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
);
continue;
}
state
.store
.write()
.unwrap()
.mount_recovered_ec_shards_only(Some(m.vid));
if current.is_some() {
if fetch_ec_index_from_peers(state, m, &peers).await {
recovered += 1;
}
}
if recovered > 0 {
state.store.write().unwrap().mount_recovered_ec_shards();
tracing::info!(
"recovered missing EC index for {} volume(s) from peers and mounted their shards",
recovered
@@ -125,11 +125,6 @@ pub struct VolumeServerState {
pub cli_white_list: Vec<String>,
/// Path to state.pb file for persisting VolumeServerState across restarts.
pub state_file_path: String,
/// Per-vid EC lifecycle lock, serializing EC ops that install or remove a
/// vid's files. A tokio Mutex so the guard survives the copy's `.await` IO.
pub ec_lifecycle_locks: std::sync::Mutex<
std::collections::HashMap<crate::storage::types::VolumeId, Arc<tokio::sync::Mutex<()>>>,
>,
/// Volumes with an EC decode in flight. A dropped request leaves the
/// blocking job running; this keeps a retry from racing it on the
/// same volume files.
@@ -156,20 +151,6 @@ pub struct VolumeServerState {
}
impl VolumeServerState {
/// Return the per-vid EC lifecycle lock, creating it on first use. Entries
/// are never reaped.
pub fn ec_lifecycle_lock(
&self,
vid: crate::storage::types::VolumeId,
) -> Arc<tokio::sync::Mutex<()>> {
self.ec_lifecycle_locks
.lock()
.unwrap()
.entry(vid)
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
/// The master lookups should go to: the leader the heartbeat confirmed,
/// or empty while none is known, in its announced `host:port[.grpcPort]`
/// form. Mirrors Go's `vs.getCurrentMaster()`.
-1
View File
@@ -467,7 +467,6 @@ mod tests {
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
+4 -57
View File
@@ -58,8 +58,6 @@ pub struct DiskLocation {
/// skips a claimed vid outright and claims any other through its load,
/// so a VolumeCopy's .note sweep cannot unlink files it still writes.
pub(crate) load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
/// The data directory of every disk on this server.
pub(crate) store_dirs: Vec<String>,
}
/// Keeps `vid` in the copy-claim set through one scan load so a VolumeCopy
@@ -125,7 +123,6 @@ impl DiskLocation {
disk_free_bytes: AtomicU64::new(0),
min_free_space,
load_guard: None,
store_dirs: Vec::new(),
})
}
@@ -489,20 +486,8 @@ impl DiskLocation {
// Custom ratio comes from the volume's own .vif; the server holds no
// cluster EC config in memory.
let data_shards = ec_data_shards_from_vif(
&self.directory,
&self.idx_directory,
&self.store_dirs,
collection,
vid,
);
if data_shards == 0 {
warn!(
volume_id = vid.0,
"sibling .vif files disagree on the encode; keeping EC shards"
);
return true;
}
let data_shards =
ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid);
// On-disk .dat: an empty <= superblock one is a stub; a transient stat
// error keeps the shards rather than deleting.
@@ -829,24 +814,6 @@ impl DiskLocation {
self.volumes.len()
}
/// Number of local volumes of `collection` that still take writes: not
/// read-only and, when `volume_size_limit` is known (non zero), with a .dat
/// below it. The .dat size is what the master compares with the limit to
/// stop writes. Mirrors Go's WritableCollectionVolumesLen.
pub fn writable_collection_volumes_len(
&self,
collection: &str,
volume_size_limit: u64,
) -> usize {
self.volumes
.values()
.filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only())
.filter(|v| {
volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit
})
.count()
}
/// Get all volume IDs, sorted.
pub fn volume_ids(&self) -> Vec<VolumeId> {
let mut ids: Vec<VolumeId> = self.volumes.keys().copied().collect();
@@ -1053,9 +1020,7 @@ impl DiskLocation {
// propagate the error to the caller.
let created = !self.ec_volumes.contains_key(&vid);
if created {
let ec_vol =
EcVolume::new_with_vif_dirs(&dir, idx_dir, &self.store_dirs, collection, vid)
.map_err(VolumeError::Io)?;
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid).map_err(VolumeError::Io)?;
self.ec_volumes.insert(vid, ec_vol);
}
let ec_vol = self.ec_volumes.get_mut(&vid).expect("just inserted above");
@@ -1476,7 +1441,6 @@ fn rm_if_present(path: String) -> io::Result<()> {
fn ec_data_shards_from_vif(
directory: &str,
idx_directory: &str,
store_dirs: &[String],
collection: &str,
vid: VolumeId,
) -> usize {
@@ -1495,23 +1459,6 @@ fn ec_data_shards_from_vif(
break;
}
}
match crate::storage::erasure_coding::ec_volume::sibling_ec_config_vif(
directory,
idx_directory,
store_dirs,
collection,
vid,
) {
Err(()) => return 0,
Ok(Some(vi)) => {
if let Some(ds) = vi.ec_shard_config.map(|c| c.data_shards as usize)
&& ds > 0
{
return ds;
}
}
Ok(None) => {}
}
DATA_SHARDS_COUNT
}
@@ -1595,7 +1542,7 @@ pub(crate) fn vif_references_remote_file(vif_path: &str) -> bool {
/// True when a `.vif` records an EC shard config, i.e. the volume was
/// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat`
/// alongside it is a leftover stub safe to remove.
pub(crate) fn vif_is_ec_volume(vif_path: &str) -> bool {
fn vif_is_ec_volume(vif_path: &str) -> bool {
fs::read_to_string(vif_path)
.ok()
.and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
@@ -272,96 +272,6 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume
data_vif
}
/// [`locate_vif_path`], except that a data-dir `.vif` recording no EC config
/// yields to the idx dir's when that one records it: a source volume's own
/// `.vif` stays on a disk that later receives shards of the same volume id,
/// while the encode's `.vif` lands with the index on a sibling.
fn resolve_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
let path = locate_vif_path(dir, dir_idx, collection, volume_id);
let data_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir, collection, volume_id),
);
if dir_idx == dir || path != data_vif {
return path;
}
let idx_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
);
prefer_ec_config_vif(path, idx_vif)
}
fn read_vif_file(path: &str) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
match std::fs::read_to_string(path) {
Ok(content) if content.trim().is_empty() => Ok(None),
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|_| ()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
Err(_) => Err(()),
}
}
/// Returns `alt` when `local` records no EC config but `alt` does.
fn prefer_ec_config_vif(local: String, alt: String) -> String {
match read_vif_file(&local) {
Ok(Some(info)) if info.ec_shard_config.is_some() => return local,
Err(()) => return local,
_ => {}
}
match read_vif_file(&alt) {
Ok(Some(info)) if info.ec_shard_config.is_some() => alt,
_ => local,
}
}
/// The `.vif` a disk mounting against a shared `-dir.idx` takes from a sibling
/// data dir in `vif_dirs`: its own records no EC config, the index is not its
/// own, and every sibling `.vif` recording an EC config names the same run.
/// `Err` when a sibling `.vif` cannot be read or the siblings name different runs.
pub(crate) fn sibling_ec_config_vif(
dir: &str,
dir_idx: &str,
vif_dirs: &[String],
collection: &str,
volume_id: VolumeId,
) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let data_vif = format!("{}.vif", data_base);
if dir_idx == dir
|| vif_dirs.iter().any(|d| d == dir_idx)
|| matches!(ecx_file_size(&format!("{}.ecx", data_base)), Some(n) if n > 0)
|| resolve_vif_path(dir, dir_idx, collection, volume_id) != data_vif
{
return Ok(None);
}
match read_vif_file(&data_vif) {
Ok(Some(info)) if info.ec_shard_config.is_some() => return Ok(None),
Err(()) => return Ok(None),
_ => {}
}
let mut found: Option<crate::storage::volume::VifVolumeInfo> = None;
for other in vif_dirs.iter().filter(|d| d.as_str() != dir) {
let path = format!(
"{}.vif",
crate::storage::volume::volume_file_name(other, collection, volume_id)
);
let info = match read_vif_file(&path) {
Ok(Some(info)) if info.ec_shard_config.is_some() => info,
Ok(_) => continue,
Err(()) => return Err(()),
};
let run = |v: &crate::storage::volume::VifVolumeInfo| {
v.ec_shard_config.as_ref().map(|c| c.encode_ts_ns)
};
match &found {
None => found = Some(info),
Some(f) if run(f) != run(&info) => return Err(()),
Some(_) => {}
}
}
Ok(found)
}
/// Load the volume's `.vif` (data dir first, then idx dir, then the active
/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
/// everywhere is legal — legacy volumes predate the sidecar — and returns
@@ -376,7 +286,7 @@ pub fn load_vif_info(
collection: &str,
volume_id: VolumeId,
) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
let vif_path = resolve_vif_path(dir, dir_idx, collection, volume_id);
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
match std::fs::read_to_string(&vif_path) {
Ok(content) if content.trim().is_empty() => Ok(None),
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
@@ -426,16 +336,15 @@ pub fn load_vif_info_across_dirs(
// pointed callers at the wrong disk whenever the vif lived with the
// index — and a caller resolving the rest of the volume's metadata
// against that answer would look in a directory holding none of it.
let idx_vif = format!(
let data_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id)
crate::storage::volume::volume_file_name(dir, collection, volume_id)
);
let found_in =
if dir_idx != dir && resolve_vif_path(dir, dir_idx, collection, volume_id) == idx_vif {
dir_idx.to_string()
} else {
dir.to_string()
};
let found_in = if std::path::Path::new(&data_vif).exists() {
dir.to_string()
} else {
dir_idx.to_string()
};
return Ok(Some((vif, found_in)));
}
for other in other_dirs {
@@ -660,25 +569,7 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL, &[])
}
/// [`Self::new`] for a disk whose server's data directories are `vif_dirs`.
pub fn new_with_vif_dirs(
dir: &str,
dir_idx: &str,
vif_dirs: &[String],
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
Self::open_with(
dir,
dir_idx,
collection,
volume_id,
EcjFsOps::REAL,
vif_dirs,
)
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL)
}
/// [`Self::new`] with the journal-compaction filesystem steps supplied, so
@@ -689,15 +580,10 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
ecj_ops: EcjFsOps,
vif_dirs: &[String],
) -> io::Result<Self> {
// One load of the volume's `.vif`, used for both the shard config and
// the version / dat-size fields below.
let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
let vif = sibling_ec_config_vif(dir, dir_idx, vif_dirs, collection, volume_id)
.ok()
.flatten()
.or(vif);
// Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
// the INDEX, and it is the only layout record a volume whose vif is
// absent or config-free still has.
@@ -3694,7 +3580,7 @@ mod tests {
);
let before = std::fs::read(&ecj_path).unwrap();
(
EcVolume::open_with(dir, dir, "", vid, ops, &[]),
EcVolume::open_with(dir, dir, "", vid, ops),
ecj_path,
before,
)
@@ -3915,7 +3801,7 @@ mod tests {
read_at: rewrite_after_last_read,
..EcjFsOps::REAL
};
let vol = EcVolume::open_with(dir, dir, "", vid, ops, &[]).unwrap();
let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap();
assert!(
IN_PLACE_REWRITE.with(|r| r.borrow().is_none()),
"the rewrite never ran"
@@ -4222,143 +4108,6 @@ mod tests {
assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
}
/// Shards placed on the disk that still holds the source volume's `.vif`
/// mount against an index on a sibling disk; the encode identity and
/// layout come from the sibling's `.vif`, not the source's config-free one.
#[test]
fn test_ec_volume_prefers_idx_vif_over_config_free_data_vif() {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
let data_base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", data_base),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
dat_file_size: 4096,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 12,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_123_456_789,
block_size: 3 * 1024 * 1024,
}),
..Default::default()
};
let idx_base = crate::storage::volume::volume_file_name(dir_idx, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", idx_base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, 1_700_000_000_123_456_789);
assert_eq!((vol.data_shards, vol.parity_shards), (12, 4));
assert_eq!(vol.block_size, 3 * 1024 * 1024);
assert_eq!(vol.dat_file_size, 4096);
let (_, found_in) = load_vif_info_across_dirs(dir, dir_idx, &[], "pics", VolumeId(1))
.unwrap()
.unwrap();
assert_eq!(found_in, dir_idx);
}
/// A data-dir `.vif` that records an EC config stays authoritative.
#[test]
fn test_ec_volume_keeps_data_vif_with_ec_config() {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
for (d, ts) in [(dir, 11), (dir_idx, 22)] {
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: ts,
..Default::default()
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(d, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
}
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, 11);
}
/// With a shared `-dir.idx` the encode's `.vif` sits in the data dir of the
/// disk that received the index, so a disk holding a source volume's
/// config-free `.vif` takes it from that sibling, but only when the
/// siblings agree on one run.
#[test]
fn test_ec_volume_takes_sibling_vif_with_shared_index() {
const RUN_A: i64 = 1_700_000_000_000_000_011;
const RUN_B: i64 = 1_700_000_000_000_000_022;
let cases: [(&str, &[i64], bool, i64); 4] = [
("one sibling", &[RUN_A], false, RUN_A),
("siblings agree", &[RUN_A, RUN_A], false, RUN_A),
("siblings disagree", &[RUN_A, RUN_B], false, 0),
("local index", &[RUN_A], true, 0),
];
for (name, sibling_ts, local_index, want) in cases {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
let entry = [(NeedleId(1), Offset::from_actual_offset(8), Size(10))];
write_ecx_file(dir_idx, "c", VolumeId(1), &entry);
if local_index {
write_ecx_file(dir, "c", VolumeId(1), &entry);
}
let data_base = crate::storage::volume::volume_file_name(dir, "c", VolumeId(1));
std::fs::write(
format!("{}.vif", data_base),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
let mut dirs = vec![dir.to_string()];
let mut siblings = Vec::new();
for &ts in sibling_ts {
let sibling = TempDir::new().unwrap();
let sibling_dir = sibling.path().to_str().unwrap().to_string();
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
dat_file_size: 4096,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: ts,
..Default::default()
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(&sibling_dir, "c", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
dirs.push(sibling_dir);
siblings.push(sibling);
}
let vol = EcVolume::new_with_vif_dirs(dir, dir_idx, &dirs, "c", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, want, "{}", name);
}
}
#[test]
fn test_ec_volume_wide_ratio_vif_config() {
let tmp = TempDir::new().unwrap();
+49 -370
View File
@@ -13,6 +13,7 @@ use std::io::{self, Read, Seek, Write};
use std::path::Path;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
#[cfg(feature = "redb-experimental-cursor")]
use std::ops::Bound;
mod compact_map;
@@ -484,38 +485,6 @@ const NEEDLE_TABLE: TableDefinition<u64, &[u8]> = TableDefinition::new("needles"
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
const META_IDX_SIZE: &str = "idx_size";
/// redb 4.6 starts a new file at 72 KiB: 64 KiB of pages plus the header
/// and the region tracker.
const REDB_SMALL_FILE_BYTES: u64 = 72 * 1024;
/// redb 4.2 started at 1 MiB of pages plus that same 8 KiB.
const REDB_OLD_FLOOR_BYTES: u64 = 1024 * 1024 + 8 * 1024;
/// Meta value `1` means a compact of a file in the old-floor band already finished.
const META_FLOOR_COMPACT: &str = "floor_compact";
/// True when a reused `.rdb` is still inside the old 4.2 starting size.
/// A file one byte past it has grown.
fn should_compact_old_floor(len: u64) -> bool {
len > REDB_SMALL_FILE_BYTES && len <= REDB_OLD_FLOOR_BYTES
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CompactFollowUp {
/// Drop the `Database` and open the same path once. Do not compact again.
ReopenSkipCompact,
/// The handle can still begin a write. Do not record `floor_compact`.
KeepHandle,
}
fn compact_follow_up(err: &redb::CompactionError) -> CompactFollowUp {
match err {
redb::CompactionError::Storage(_) => CompactFollowUp::ReopenSkipCompact,
redb::CompactionError::TransactionInProgress
| redb::CompactionError::PersistentSavepointExists
| redb::CompactionError::EphemeralSavepointExists => CompactFollowUp::KeepHandle,
_ => CompactFollowUp::KeepHandle,
}
}
/// Writes between two durable redb checkpoints. Every non-durable commit
/// leaves an entry in redb's transaction tracker and pins the pages that
/// were on disk at the last durable commit; only a durable commit clears
@@ -679,6 +648,9 @@ impl RedbNeedleMap {
let meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
// ReadableTable::get guard borrows `meta`; edition 2024 drops the tail
// expression's temporaries before `meta`, so no extra binding is needed.
match meta.get(META_IDX_SIZE) {
Ok(Some(guard)) => Ok(Some(guard.value())),
Ok(None) => Ok(None),
@@ -686,118 +658,6 @@ impl RedbNeedleMap {
}
}
fn rdb_file_len(&self) -> io::Result<u64> {
Ok(std::fs::metadata(&self.rdb_path)?.len())
}
fn read_floor_compact(&self) -> io::Result<Option<u64>> {
let txn = self
.db_or_err()?
.begin_read()
.map_err(|e| io::Error::other(format!("redb begin_read: {}", e)))?;
let meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
match meta.get(META_FLOOR_COMPACT) {
Ok(Some(guard)) => Ok(Some(guard.value())),
Ok(None) => Ok(None),
Err(e) => Err(io::Error::other(format!("redb get meta: {}", e))),
}
}
fn save_floor_compact(&self) -> io::Result<()> {
let txn = Self::begin_write_no_fsync(self.db_or_err()?)?;
{
let mut meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
meta.insert(META_FLOOR_COMPACT, 1u64)
.map_err(|e| io::Error::other(format!("redb insert meta: {}", e)))?;
}
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit meta: {}", e)))?;
Ok(())
}
/// Drop the exclusive lock and open the same file. Does not compact.
fn reopen_database(&mut self) -> io::Result<()> {
self.db.take();
let db = Database::builder()
.set_cache_size(self.cache_bytes)
.open(&self.rdb_path)
.map_err(|e| io::Error::other(format!("redb reopen after compact: {}", e)))?;
self.db = Some(db);
Ok(())
}
fn record_floor_compact_if_still_in_band(&self) -> io::Result<()> {
let len = match self.rdb_file_len() {
Ok(len) => len,
Err(e) => {
tracing::warn!("redb floor compact flag skipped, metadata failed: {}", e);
return Ok(());
}
};
if should_compact_old_floor(len) {
self.save_floor_compact()?;
}
Ok(())
}
/// Compact a reused file that is still the 4.2 starting allocation.
///
/// Called only from `try_reuse_rdb`, after the replay transaction has
/// committed. `new` and `full_rebuild` already create a 72 KiB file.
///
/// An empty durable commit publishes a `Durability::None` tail before
/// `compact`. A storage error inside `compact` would otherwise drop that
/// tail when the handle is reopened. `checkpoint` is not used: during
/// reuse `idx_file` is `None` and `idx_file_offset` is 0.
fn shrink_old_floor(&mut self) -> io::Result<()> {
let len = match self.rdb_file_len() {
Ok(len) => len,
Err(e) => {
tracing::warn!("redb floor compact skipped, metadata failed: {}", e);
return Ok(());
}
};
// The flag is irrelevant outside the band. A meta read error there
// must not make `load_from_idx` rebuild a larger index.
if !should_compact_old_floor(len) {
return Ok(());
}
if self.read_floor_compact()? == Some(1) {
return Ok(());
}
// Failure returns Err so `load_from_idx` full-rebuilds. Do not reopen
// over a tail that this commit did not publish.
let mut txn = self
.db_or_err()?
.begin_write()
.map_err(|e| io::Error::other(format!("redb begin_write: {}", e)))?;
txn.set_quick_repair(true);
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
let compact_result = self
.db
.as_mut()
.ok_or_else(|| io::Error::other("redb database is closed"))?
.compact();
match compact_result {
Ok(_) => self.record_floor_compact_if_still_in_band(),
Err(e) => match compact_follow_up(&e) {
CompactFollowUp::ReopenSkipCompact => {
tracing::warn!("redb floor compact storage error, reopening: {}", e);
self.reopen_database()
}
CompactFollowUp::KeepHandle => {
tracing::warn!("redb floor compact skipped: {}", e);
Ok(())
}
},
}
}
/// Test-only read of META `idx_size` through the live handle. See
/// [`test_support::live_meta_idx_size`] for why durability tests use
/// this instead of copying the open `.rdb`.
@@ -806,11 +666,6 @@ impl RedbNeedleMap {
self.read_idx_size_meta().unwrap()
}
#[cfg(test)]
fn live_floor_compact(&self) -> Option<u64> {
self.read_floor_compact().unwrap()
}
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
///
/// Strategy:
@@ -919,7 +774,6 @@ impl RedbNeedleMap {
nm.save_idx_size_meta(idx_size)?;
}
nm.shrink_old_floor()?;
Ok(nm)
}
@@ -942,8 +796,8 @@ impl RedbNeedleMap {
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
/// still writable, clear the leftover needles table before inserting.
/// Live keys come from a `BTreeMap` so `insert_before` walks them in
/// needle-id order and packs leaves, without a second Vec + sort scratch.
/// Live keys come from a `BTreeMap` so inserts are in needle-id order and
/// redb 4.2.0 packs leaves, without a second Vec + sort scratch.
fn full_rebuild<R: Read + Seek>(
db_path: &str,
reader: &mut R,
@@ -995,26 +849,43 @@ impl RedbNeedleMap {
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
}
let mut cursor = table
.upper_bound_mut(Bound::<u64>::Unbounded)
.map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb upper_bound_mut: {}", e))
#[cfg(not(feature = "redb-experimental-cursor"))]
{
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
table
.insert(key_u64, packed.as_slice())
.map_err(|e| io::Error::other(format!("redb insert: {}", e)))?;
}
}
#[cfg(feature = "redb-experimental-cursor")]
{
let mut cursor =
table
.upper_bound_mut(Bound::<u64>::Unbounded)
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb upper_bound_mut: {}", e),
)
})?;
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
cursor
.insert_before(key_u64, packed.as_slice())
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb insert_before: {}", e),
)
})?;
}
cursor.close().map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
})?;
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
cursor
.insert_before(key_u64, packed.as_slice())
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb insert_before: {}", e),
)
})?;
}
cursor.close().map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
})?;
}
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
@@ -1177,6 +1048,9 @@ impl RedbNeedleMap {
let table = txn
.open_table(NEEDLE_TABLE)
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
// ReadableTable::get guard borrows `table`; edition 2024 drops the tail
// expression's temporaries before `table`, so no extra binding is needed.
match table.get(key_u64) {
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
Ok(None) => Ok(None),
@@ -1307,7 +1181,7 @@ impl RedbNeedleMap {
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
/// commit error.
///
/// redb can make a `Durability::None` commit visible *before*
/// redb 4.2.0 can make a `Durability::None` commit visible *before*
/// returning `Err(CommitError::Storage(..))`. In that state the database
/// refuses further write transactions, so the map cannot be used as-is.
/// The `.idx` row that triggered the commit is **not** truncated — the
@@ -1729,10 +1603,9 @@ pub(crate) mod test_support {
/// every `put`/`delete` also commits non-durably, so before the first
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
/// A live read is the only portable observation: redb's default
/// `ExclusiveWriter` takes an exclusive whole-file lock, which is
/// advisory on Unix but mandatory on Windows, so copying the open
/// `.rdb` fails there with OS error 33.
/// A live read is the only portable observation: redb 4.2.0 takes an
/// exclusive whole-file lock, which is advisory on Unix but mandatory
/// on Windows, so copying the open `.rdb` fails there with OS error 33.
///
/// It still pins the property under test: the only *durable* META
/// writer is `checkpoint`, so any value other than the load-time one
@@ -1890,41 +1763,6 @@ mod tests {
NeedleMapKind::Redb.redb_cache_bytes()
}
#[test]
fn test_redb_new_file_is_at_most_72kib() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("fresh.rdb");
let nm = RedbNeedleMap::new(db_path.to_str().unwrap(), redb_test_cache()).unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
assert!(
len <= REDB_SMALL_FILE_BYTES,
"fresh redb file is {len} bytes, above the 72 KiB start"
);
drop(nm);
}
#[test]
fn test_should_compact_old_floor_bounds() {
let small = REDB_SMALL_FILE_BYTES;
let floor = REDB_OLD_FLOOR_BYTES;
assert!(!should_compact_old_floor(small));
assert!(should_compact_old_floor(small + 1));
assert!(should_compact_old_floor(floor));
assert!(!should_compact_old_floor(floor + 1));
assert!(!should_compact_old_floor(0));
}
fn reload_redb(db_path: &std::path::Path, idx_path: &std::path::Path) -> RedbNeedleMap {
let mut idx = std::fs::File::open(idx_path).unwrap();
RedbNeedleMap::load_from_idx(
db_path.to_str().unwrap(),
&mut idx,
Version::current(),
redb_test_cache(),
)
.unwrap()
}
/// Open a redb map on an empty .idx with an append writer attached, the
/// way `Volume::load_index_redb` opens a writable volume.
fn open_writable_redb(
@@ -2057,6 +1895,7 @@ mod tests {
);
}
#[cfg(feature = "redb-experimental-cursor")]
#[test]
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
// Enough keys to split leaves. insert_before vs table.insert only
@@ -2635,164 +2474,4 @@ mod tests {
);
}
}
#[test]
fn test_redb_fresh_reuse_does_not_record_floor_compact() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let len = std::fs::metadata(&db_path).unwrap().len();
assert!(len <= REDB_SMALL_FILE_BYTES, "fresh file is {len} bytes");
nm.close();
drop(nm);
let loaded = reload_redb(&db_path, &idx_path);
assert_eq!(loaded.live_floor_compact(), None);
assert!(loaded.get(NeedleId(1)).unwrap().is_none());
}
#[test]
fn test_redb_in_band_file_is_compacted_once() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let mut n = 0u64;
loop {
n += 1;
nm.put(
NeedleId(n),
Offset::from_actual_offset((n as i64).saturating_mul(8)),
Size(8),
)
.unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
if should_compact_old_floor(len) {
break;
}
assert!(
n < 2_000_000,
"file grew to {len} bytes without entering the old-floor band"
);
}
nm.checkpoint(true).unwrap();
nm.close();
drop(nm);
let mut loaded = reload_redb(&db_path, &idx_path);
let len = std::fs::metadata(&db_path).unwrap().len();
let flag = loaded.live_floor_compact();
if should_compact_old_floor(len) {
assert_eq!(flag, Some(1));
} else {
assert!(len <= REDB_SMALL_FILE_BYTES, "compacted file is {len} bytes");
assert_eq!(flag, None);
}
let first = loaded.get(NeedleId(1)).unwrap();
assert!(first.is_some());
let last = loaded.get(NeedleId(n)).unwrap();
assert!(last.is_some());
// close checkpoints the non-durable floor_compact flag, then drop
// releases the exclusive lock before the second open.
loaded.close();
drop(loaded);
let again = reload_redb(&db_path, &idx_path);
assert_eq!(std::fs::metadata(&db_path).unwrap().len(), len);
assert_eq!(again.live_floor_compact(), flag);
}
#[test]
fn test_redb_sparse_old_floor_shrinks_on_reuse() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let mut n = 0u64;
loop {
n += 1;
nm.put(
NeedleId(n),
Offset::from_actual_offset((n as i64).saturating_mul(8)),
Size(8),
)
.unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
if should_compact_old_floor(len) {
break;
}
assert!(n < 2_000_000, "file is {len} bytes and never entered the band");
}
assert!(n > 32, "band was reached with only {n} keys");
nm.checkpoint(true).unwrap();
nm.close();
drop(nm);
let keep = 32u64;
let idx = std::fs::OpenOptions::new().write(true).open(&idx_path).unwrap();
idx.set_len(keep * NEEDLE_MAP_ENTRY_SIZE as u64).unwrap();
drop(idx);
let db = redb::Database::builder()
.set_cache_size(redb_test_cache())
.open(db_path.to_str().unwrap())
.unwrap();
let txn = db.begin_write().unwrap();
{
let mut table = txn.open_table(NEEDLE_TABLE).unwrap();
for key in (keep + 1)..=n {
table.remove(key).unwrap();
}
}
{
let mut meta = txn.open_table(META_TABLE).unwrap();
meta.insert(META_IDX_SIZE, keep * NEEDLE_MAP_ENTRY_SIZE as u64)
.unwrap();
}
txn.commit().unwrap();
drop(db);
let before = std::fs::metadata(&db_path).unwrap().len();
let loaded = reload_redb(&db_path, &idx_path);
let after = std::fs::metadata(&db_path).unwrap().len();
assert!(after < before, "reuse left the file at {after} bytes, was {before}");
assert!(
after <= REDB_SMALL_FILE_BYTES,
"32 live keys still occupy {after} bytes"
);
assert!(loaded.get(NeedleId(1)).unwrap().is_some());
assert!(loaded.get(NeedleId(keep)).unwrap().is_some());
assert!(loaded.get(NeedleId(keep + 1)).unwrap().is_none());
}
#[test]
fn test_compact_follow_up_reopens_only_on_storage_error() {
use redb::{CompactionError, StorageError};
let storage = CompactionError::Storage(StorageError::Io(std::io::Error::other("sync")));
assert_eq!(
compact_follow_up(&storage),
CompactFollowUp::ReopenSkipCompact
);
assert_eq!(
compact_follow_up(&CompactionError::TransactionInProgress),
CompactFollowUp::KeepHandle
);
assert_eq!(
compact_follow_up(&CompactionError::PersistentSavepointExists),
CompactFollowUp::KeepHandle
);
assert_eq!(
compact_follow_up(&CompactionError::EphemeralSavepointExists),
CompactFollowUp::KeepHandle
);
}
#[test]
fn test_redb_reopen_database_reads_committed_key() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, _db_path, _idx_path) = open_writable_redb(dir.path());
nm.put(NeedleId(7), Offset::from_actual_offset(64), Size(9))
.unwrap();
nm.reopen_database().unwrap();
let nv = nm.get(NeedleId(7)).unwrap().unwrap();
assert_eq!(nv.size, Size(9));
nm.put(NeedleId(8), Offset::from_actual_offset(128), Size(10))
.unwrap();
let nv = nm.get(NeedleId(8)).unwrap().unwrap();
assert_eq!(nv.size, Size(10));
}
}
+14 -118
View File
@@ -122,9 +122,6 @@ pub struct Store {
/// The copy-claim set installed on every location by
/// set_volume_load_guard, kept here so a later add_location gets it too.
load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
/// The data directory of every disk this server loads, set before the
/// first `add_location` so a disk's EC mounts can see its siblings.
pub store_dirs: Vec<String>,
}
impl Store {
@@ -151,7 +148,6 @@ impl Store {
disk_stats_of: super::disk_location::get_disk_stats,
load_guard: None,
preallocate_supported: PREALLOCATE_SUPPORTED,
store_dirs: Vec::new(),
}
}
@@ -174,7 +170,6 @@ impl Store {
tags,
)?;
loc.load_guard = self.load_guard.clone();
loc.store_dirs = self.store_dirs.clone();
loc.load_existing_volumes(self.needle_map_kind)?;
// Check for duplicate volume IDs across existing locations
@@ -292,20 +287,12 @@ impl Store {
// ---- Volume lifecycle ----
/// Find the location with the fewest writable volumes of `collection`, so
/// they do not share one disk's write bandwidth, then the most free slots
/// (load-balance) of the given disk type. Matches Go's FindFreeLocation:
/// accounts for EC shards when computing free slots.
fn find_free_location(
&self,
disk_type: &DiskType,
collection: &str,
preallocate: u64,
) -> Option<usize> {
/// Find the location with fewest volumes (load-balance) of the given disk type.
/// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots.
fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option<usize> {
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed);
let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free)
let mut best: Option<(usize, i64)> = None; // (index, effective_free)
for (i, loc) in self.locations.iter().enumerate() {
if &loc.disk_type != disk_type {
continue;
@@ -335,14 +322,11 @@ impl Store {
) {
continue;
}
let writable = loc.writable_collection_volumes_len(collection, volume_size_limit);
if best.is_none_or(|(_, min_writable, max_free)| {
writable < min_writable || (writable == min_writable && effective_free > max_free)
}) {
best = Some((i, writable, effective_free));
if best.is_none() || effective_free > best.unwrap().1 {
best = Some((i, effective_free));
}
}
best.map(|(i, _, _)| i)
best.map(|(i, _)| i)
}
/// Find a free location matching a predicate.
@@ -612,7 +596,7 @@ impl Store {
return Err(VolumeError::AlreadyExists);
}
let loc_idx = self
.find_free_location(&disk_type, spec.collection, spec.preallocate)
.find_free_location(&disk_type, spec.preallocate)
.ok_or_else(|| {
VolumeError::Io(io::Error::other(format!(
"no free location for disk type {:?}",
@@ -1053,18 +1037,13 @@ impl Store {
}
/// Delete a needle from a volume.
pub fn delete_volume_needle(
&self,
vid: VolumeId,
n: &mut Needle,
fsync: bool,
) -> Result<Size, VolumeError> {
pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result<Size, VolumeError> {
// Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
if vol.is_no_write_or_delete() {
return Err(VolumeError::ReadOnly(vid));
}
vol.delete_needle(n, fsync)
vol.delete_needle(n)
}
// ---- Collection operations ----
@@ -2939,9 +2918,7 @@ mod tests {
cookie: Cookie(0xaa),
..Needle::default()
};
let deleted = store
.delete_volume_needle(VolumeId(1), &mut del_n, false)
.unwrap();
let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap();
assert!(deleted.0 > 0);
}
@@ -3050,87 +3027,6 @@ mod tests {
assert_eq!(store.locations[1].volumes_len(), 1);
}
fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) {
let spec = VolumeSpec {
collection,
..VolumeSpec::default()
};
store
.add_volume(VolumeId(vid), DiskType::HardDrive, &spec)
.unwrap();
}
fn location_of(store: &Store, vid: u32) -> Option<usize> {
store
.locations
.iter()
.position(|loc| loc.find_volume(VolumeId(vid)).is_some())
}
// Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks.
#[test]
fn test_add_volume_spreads_collection_over_disks() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
add_spread_test_volume(&mut store, 1, "other");
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
let first = location_of(&store, 3);
// Leave the disk with the "spread" volume holding fewer volumes overall.
let other_on_first = if first == Some(1) { 2 } else { 1 };
store
.delete_volume(VolumeId(other_on_first), false, false, false)
.unwrap();
add_spread_test_volume(&mut store, 4, "spread");
assert_ne!(location_of(&store, 4), first);
}
// Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection.
#[test]
fn test_add_volume_ignores_full_volumes_of_collection() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
// Every .dat holds at least its superblock, so all volumes are full.
store.volume_size_limit.store(1, Ordering::Relaxed);
add_spread_test_volume(&mut store, 1, "spread");
let full_disk = location_of(&store, 1);
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
assert_eq!(location_of(&store, 3), full_disk);
}
// Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection.
#[test]
fn test_add_volume_ignores_read_only_volumes_of_collection() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
add_spread_test_volume(&mut store, 1, "spread");
let writable_disk = location_of(&store, 1);
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
assert_ne!(location_of(&store, 3), writable_disk);
store
.find_volume(VolumeId(3))
.unwrap()
.1
.set_read_only()
.unwrap();
// The other disk holds more volumes, but its "spread" volume is read-only.
add_spread_test_volume(&mut store, 4, "spread");
assert_ne!(location_of(&store, 4), writable_disk);
}
#[test]
fn test_store_delete_collection() {
let tmp = TempDir::new().unwrap();
@@ -3499,7 +3395,7 @@ mod tests {
cookie: Cookie(2),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap();
@@ -3823,7 +3719,7 @@ mod tests {
cookie: Cookie(2),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
let revision = {
let (_, v) = store.find_volume(vid).unwrap();
v.super_block().compaction_revision
@@ -3848,7 +3744,7 @@ mod tests {
cookie: Cookie(4),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
drop(release_tx);
copy.join().unwrap().unwrap();
+12 -270
View File
@@ -8,9 +8,7 @@ use std::path::Path;
use tracing::{info, warn};
use crate::storage::disk_location::{
DiskLocation, parse_collection_volume_id_pub, vif_is_ec_volume,
};
use crate::storage::disk_location::{DiskLocation, parse_collection_volume_id_pub};
use crate::storage::store::Store;
use crate::storage::types::VolumeId;
@@ -36,14 +34,10 @@ impl Store {
/// `reconcile_ec_shards_across_disks` so the orphan pass can
/// prefer the local idx_directory.
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
self.mirror_ec_metadata_to_shard_disks_only(None)
}
pub(crate) fn mirror_ec_metadata_to_shard_disks_only(&mut self, only: Option<VolumeId>) {
if self.locations.len() < 2 {
return;
}
let owners = self.index_ecx_owners_for_mirror(only);
let owners = self.index_ecx_owners_for_mirror();
if owners.is_empty() {
return;
}
@@ -58,7 +52,7 @@ impl Store {
}
let mut mirrors: Vec<Mirror> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let orphans = collect_shard_disk_volumes(loc, only);
let orphans = collect_shard_disk_volumes(loc);
for (key, _shards) in orphans {
let Some(owner) = owners.get(&key) else {
continue;
@@ -117,7 +111,7 @@ impl Store {
// Records both idx_dir (where .ecx was found) and data_dir, so
// the mirror can resolve .vif from data_dir even when .ecx lives
// in idx_directory.
fn index_ecx_owners_for_mirror(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwner> {
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let mut seen: Vec<&str> = Vec::with_capacity(2);
@@ -137,18 +131,15 @@ impl Store {
let Some(base) = name.strip_suffix(".ecx") else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
// credible owner — skip it so the scan keeps looking for a
// real index on a sibling disk (Go's indexEcxOwners).
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
owners
.entry(EcKey { collection, vid })
.or_insert_with(|| EcxOwner {
@@ -216,33 +207,6 @@ fn mirror_sidecars_for_volume(
collection: &str,
vid: VolumeId,
) -> io::Result<usize> {
// A local .vif with no EC config (a source volume's own) would shadow the
// owner's once the index is copied next to it; keep mounting across disks.
let dst_vif = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ".vif");
if path_is_regular_file(&dst_vif)
&& !vif_is_ec_volume(&dst_vif)
&& (vif_is_ec_volume(&sidecar_dest_path(
src_data_dir,
src_idx_dir,
collection,
vid,
".vif",
)) || vif_is_ec_volume(&sidecar_dest_path(
src_idx_dir,
src_data_dir,
collection,
vid,
".vif",
)))
{
info!(
volume_id = vid.0,
collection = %collection,
vif = %dst_vif,
"local .vif records no EC config; not mirroring EC sidecars",
);
return Ok(0);
}
let mut copied = 0usize;
for ext in EC_MIRRORED_SIDECARS {
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
@@ -304,10 +268,7 @@ fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
Ok(())
}
fn collect_shard_disk_volumes(
loc: &DiskLocation,
only: Option<VolumeId>,
) -> HashMap<EcKey, Vec<String>> {
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
let Ok(read) = fs::read_dir(&loc.directory) else {
return out;
@@ -324,16 +285,13 @@ fn collect_shard_disk_volumes(
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
match ent.metadata() {
Ok(meta) if meta.len() > 0 => {}
_ => continue,
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
out.entry(EcKey { collection, vid }).or_default().push(name);
}
out
@@ -433,222 +391,6 @@ mod tests {
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
}
/// Shards on a disk that still holds the source volume's config-free
/// `.vif` must mount with the owner's encode identity: mirroring the index
/// next to that `.vif` would make the disk mount through it.
#[test]
fn mirror_skips_disk_with_config_free_vif() {
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
fs::create_dir_all(&dir0).unwrap();
fs::create_dir_all(&dir1).unwrap();
let collection = "c";
let vid = 4122u32;
plant_shard(&dir0, collection, vid, 2);
plant_shard(&dir1, collection, vid, 0);
fs::write(
dir0.join(format!("{}_{}.vif", collection, vid)),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_000_000_042,
..Default::default()
}),
..Default::default()
};
fs::write(
dir1.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
add_loc(&mut store, &dir0);
add_loc(&mut store, &dir1);
assert!(
!dir0.join(format!("{}_{}.ecx", collection, vid)).exists(),
".ecx mirrored next to the config-free .vif"
);
let ec_vol = store.locations[0]
.find_ec_volume(VolumeId(vid))
.expect("dir0 ec volume mounted");
assert_eq!(ec_vol.encode_ts_ns, 1_700_000_000_000_000_042);
}
/// With a shared `-dir.idx` every disk sees the `.ecx`, and only the disk
/// that received it holds the encode's `.vif`. A disk loaded first that
/// still holds the source volume's config-free `.vif` must take that
/// identity from its sibling.
#[test]
fn shared_index_disk_takes_sibling_vif_on_startup() {
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
let idx = tmp.path().join("idx");
for d in [&dir0, &dir1, &idx] {
fs::create_dir_all(d).unwrap();
}
let collection = "c";
let vid = 4123u32;
plant_shard(&dir0, collection, vid, 2);
plant_shard(&dir1, collection, vid, 0);
fs::write(
dir0.join(format!("{}_{}.vif", collection, vid)),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_000_000_043,
..Default::default()
}),
..Default::default()
};
fs::write(
dir1.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
store.store_dirs = vec![
dir0.to_str().unwrap().to_string(),
dir1.to_str().unwrap().to_string(),
];
for dir in [&dir0, &dir1] {
store
.add_location(
dir.to_str().unwrap(),
idx.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
for loc in &store.locations {
let ec_vol = loc
.find_ec_volume(VolumeId(vid))
.unwrap_or_else(|| panic!("{} ec volume mounted", loc.directory));
assert_eq!(
ec_vol.encode_ts_ns, 1_700_000_000_000_000_043,
"{}",
loc.directory
);
}
}
/// Starts a store whose first disk still holds the source `.dat`, its
/// config-free `.vif` and one 12+4 shard, with the `.ecx` in a shared
/// index dir and one sibling encode `.vif` per entry of `sibling_runs`.
/// Asserts the source `.dat`, the shard and the `.ecx` survive startup.
fn start_with_sibling_encode_vifs(sibling_runs: &[i64]) -> Store {
let tmp = TempDir::new().unwrap();
let dirs: Vec<_> = (0..=sibling_runs.len())
.map(|i| tmp.path().join(format!("data{}", i)))
.collect();
let idx = tmp.path().join("idx");
for d in dirs.iter().chain([&idx]) {
fs::create_dir_all(d).unwrap();
}
let collection = "c";
let vid = 4124u32;
let base0 = dirs[0].join(format!("{}_{}", collection, vid));
let dat = format!("{}.dat", base0.display());
let shard = format!("{}.ec02", base0.display());
fs::File::create(&dat)
.unwrap()
.set_len(11 * 1024 * 1024)
.unwrap();
fs::File::create(&shard)
.unwrap()
.set_len(1024 * 1024)
.unwrap();
fs::write(
format!("{}.vif", base0.display()),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
for (dir, run) in dirs[1..].iter().zip(sibling_runs) {
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 12,
parity_shards: 4,
encode_ts_ns: *run,
..Default::default()
}),
..Default::default()
};
fs::write(
dir.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
}
let mut store = Store::new(NeedleMapKind::InMemory);
store.store_dirs = dirs
.iter()
.map(|d| d.to_str().unwrap().to_string())
.collect();
for dir in &dirs {
store
.add_location(
dir.to_str().unwrap(),
idx.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
let ecx = idx.join(format!("{}_{}.ecx", collection, vid));
for path in [dat.as_str(), shard.as_str(), ecx.to_str().unwrap()] {
assert!(Path::new(path).exists(), "{} removed at startup", path);
}
store
}
/// A disk still holding the source `.dat` sizes its shards by the
/// sibling's encode `.vif` before the startup check decides whether they
/// are leftovers of an interrupted encode.
#[test]
fn shared_index_startup_validation_uses_sibling_vif_ratio() {
let store = start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044]);
let ec_vol = store.locations[0]
.find_ec_volume(VolumeId(4124))
.expect("dir0 ec volume mounted");
assert!(ec_vol.has_shard(2), "shard 2 not mounted on dir0");
}
/// Sibling `.vif` files naming different encode runs leave the ratio
/// unknown, so the startup check keeps the shards.
#[test]
fn shared_index_startup_validation_keeps_shards_on_disagreeing_siblings() {
start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044, 1_700_000_000_000_000_045]);
}
#[test]
fn mirror_preserves_existing_destination_sidecars() {
let tmp = TempDir::new().unwrap();
@@ -701,7 +443,7 @@ mod tests {
add_loc(&mut store, &dir0);
add_loc(&mut store, &dir1);
let owners = store.index_ecx_owners_for_mirror(None);
let owners = store.index_ecx_owners_for_mirror();
let owner = owners
.get(&EcKey {
collection: collection.to_string(),
+18 -115
View File
@@ -92,15 +92,11 @@ impl Store {
///
/// Mirrors `Store.reconcileEcShardsAcrossDisks` in Go.
pub fn reconcile_ec_shards_across_disks(&mut self) {
self.reconcile_ec_shards_across_disks_only(None)
}
pub(crate) fn reconcile_ec_shards_across_disks_only(&mut self, only: Option<VolumeId>) {
if self.locations.len() < 2 {
return;
}
let owners = self.index_ecx_owners(only);
let owners = self.index_ecx_owners();
if owners.is_empty() {
return;
}
@@ -110,7 +106,7 @@ impl Store {
// loc.idx_directory instead of the owner disk.
let mut to_load: Vec<OrphanShardLoad> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let orphans = collect_orphan_ec_shards(loc, loc_idx, only);
let orphans = collect_orphan_ec_shards(loc, loc_idx);
for (key, shards) in orphans {
let Some(owner) = owners.get(&key) else {
warn!(
@@ -406,7 +402,7 @@ impl Store {
/// matched so downstream loaders point `EcVolume::new` at the
/// directory that really has the file. The first owner found wins;
/// duplicates across disks are unusual but tolerated.
fn index_ecx_owners(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwnerInfo> {
fn index_ecx_owners(&self) -> HashMap<EcKey, EcxOwnerInfo> {
let mut owners: HashMap<EcKey, EcxOwnerInfo> = HashMap::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let mut seen: Vec<&str> = Vec::with_capacity(2);
@@ -426,18 +422,15 @@ impl Store {
let Some(base) = name.strip_suffix(".ecx") else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
// credible owner — skip it so the scan keeps looking for a
// real index on a sibling disk (Go's indexEcxOwners).
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
let key = EcKey { collection, vid };
owners.entry(key).or_insert(EcxOwnerInfo {
location: loc_idx,
@@ -457,18 +450,11 @@ impl Store {
///
/// Mirrors `Store.CollectEcVolumesMissingIndex` in Go.
pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec<EcVolumeMissingIndex> {
self.collect_ec_volumes_missing_index_only(None)
}
pub(crate) fn collect_ec_volumes_missing_index_only(
&self,
only: Option<VolumeId>,
) -> Vec<EcVolumeMissingIndex> {
let owners = self.index_ecx_owners(only);
let owners = self.index_ecx_owners();
let mut seen: HashSet<EcKey> = HashSet::new();
let mut missing = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) {
if owners.contains_key(&key) || !seen.insert(key.clone()) {
continue;
}
@@ -488,24 +474,18 @@ impl Store {
/// the disks that now have a local index, then fall back to the cross-disk
/// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go.
pub(crate) fn mount_recovered_ec_shards(&mut self) {
self.mount_recovered_ec_shards_only(None)
}
/// `mount_recovered_ec_shards` restricted to one volume's shards when
/// `only` names it.
pub(crate) fn mount_recovered_ec_shards_only(&mut self, only: Option<VolumeId>) {
self.mirror_ec_metadata_to_shard_disks_only(only);
self.load_orphan_ec_shards_with_local_index(only);
self.reconcile_ec_shards_across_disks_only(only);
self.mirror_ec_metadata_to_shard_disks();
self.load_orphan_ec_shards_with_local_index();
self.reconcile_ec_shards_across_disks();
}
/// Mount on-disk EC shards whose `.ecx` index is now present on the same disk.
/// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a
/// single-disk store recovers once its index has been fetched from a peer.
fn load_orphan_ec_shards_with_local_index(&mut self, only: Option<VolumeId>) {
fn load_orphan_ec_shards_with_local_index(&mut self) {
let mut work: Vec<(usize, EcKey, Vec<ShardId>)> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) {
if !loc.has_ecx_file_on_disk(&key.collection, key.vid) {
continue;
}
@@ -536,7 +516,6 @@ impl Store {
fn collect_orphan_ec_shards(
loc: &crate::storage::disk_location::DiskLocation,
_loc_idx: usize,
only: Option<VolumeId>,
) -> HashMap<EcKey, Vec<(String, ShardId)>> {
let mut orphans: HashMap<EcKey, Vec<(String, ShardId)>> = HashMap::new();
let Ok(read) = fs::read_dir(&loc.directory) else {
@@ -554,12 +533,6 @@ fn collect_orphan_ec_shards(
let Some(shard_id) = is_ec_shard_extension(ext) else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// Ignore zero-byte shards. Use the DirEntry's metadata so we
// don't pay a second stat syscall per file beyond what
// read_dir already returned.
@@ -567,6 +540,9 @@ fn collect_orphan_ec_shards(
Ok(meta) if meta.len() > 0 => {}
_ => continue,
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
// Skip shards that are already registered to an EcVolume.
if let Some(ecv) = loc.find_ec_volume(vid)
&& ecv.has_shard(shard_id)
@@ -679,7 +655,7 @@ mod tests {
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
write_index_files(&d1, "pics", 7, 10, 4);
let owners = store.index_ecx_owners(None);
let owners = store.index_ecx_owners();
let owner = owners
.get(&EcKey {
collection: "pics".to_string(),
@@ -691,7 +667,7 @@ mod tests {
// A stub with no real index anywhere owns nothing.
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
assert!(!store.index_ecx_owners(None).contains_key(&EcKey {
assert!(!store.index_ecx_owners().contains_key(&EcKey {
collection: "pics".to_string(),
vid: VolumeId(8),
}));
@@ -1264,79 +1240,6 @@ mod tests {
);
}
#[test]
fn test_mount_recovered_ec_shards_only_named_volume() {
// Recovering one volume mounts only its shards. Another volume's orphan
// shards, such as ones an in-flight copy just renamed in, stay unmounted
// even when an index for them is already on disk. Mirrors Go
// TestMountRecoveredEcShards_OnlyNamedVolume.
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
std::fs::create_dir_all(&dir0).unwrap();
std::fs::create_dir_all(&dir1).unwrap();
let collection = "col";
let (recovered_vid, other_vid) = (71u32, 72u32);
let mut store = Store::new(NeedleMapKind::InMemory);
for d in [&dir0, &dir1] {
store
.add_location(
d.to_str().unwrap(),
d.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
for vid in [recovered_vid, other_vid] {
write_shard(dir0.to_str().unwrap(), collection, vid, 0);
write_shard(dir1.to_str().unwrap(), collection, vid, 1);
}
write_index_files(dir0.to_str().unwrap(), collection, other_vid, 10, 4);
let missing = store.collect_ec_volumes_missing_index();
let m = missing
.iter()
.find(|m| m.vid == VolumeId(recovered_vid))
.expect("recovered volume not reported as missing its index");
write_index_files(&m.idx_dir, collection, recovered_vid, 10, 4);
store.mount_recovered_ec_shards_only(Some(VolumeId(recovered_vid)));
for (i, loc) in store.locations.iter().enumerate() {
let ev = loc
.find_ec_volume(VolumeId(recovered_vid))
.unwrap_or_else(|| panic!("volume {} not mounted on disk{}", recovered_vid, i));
assert!(
ev.has_shard(i as u8),
"shard {}.{} not mounted",
recovered_vid,
i
);
assert!(
loc.find_ec_volume(VolumeId(other_vid)).is_none(),
"volume {} mounted on disk{} by the recovery of volume {}",
other_vid,
i,
recovered_vid,
);
}
let mirrored = format!(
"{}/{}_{}.ecx",
dir1.to_str().unwrap(),
collection,
other_vid
);
assert!(
!std::path::Path::new(&mirrored).exists(),
"volume {} index mirrored to disk1 by the recovery of volume {}",
other_vid,
recovered_vid,
);
}
#[test]
fn test_collect_missing_index_excludes_cross_disk_orphan() {
// A .ecx merely on a sibling disk is the same-server case the reconcile
File diff suppressed because it is too large. Load diff
@@ -188,7 +188,7 @@ mod tests {
for id in 1..=3 {
v.write_needle(&mut needle(id), true, false).unwrap();
}
v.delete_needle(&mut needle(2), false).unwrap();
v.delete_needle(&mut needle(2)).unwrap();
v.sync_to_disk().unwrap();
let (want_count, want_deleted) = (v.file_count(), v.deleted_count());
drop(v);
@@ -382,13 +382,10 @@ pub(crate) mod tests {
write_test_volume(dir, 6);
let v = open_volume(dir);
v.delete_needle(
&mut Needle {
id: NeedleId(3),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(3),
..Needle::default()
})
.unwrap();
drop(v);
@@ -420,13 +417,10 @@ pub(crate) mod tests {
write_test_volume(dir, 8);
let v = open_volume(dir);
v.delete_needle(
&mut Needle {
id: NeedleId(2),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(2),
..Needle::default()
})
.unwrap();
drop(v);
@@ -505,13 +505,10 @@ mod tests {
f.written.insert(*id, n.data.clone());
}
Op::Del(id) => {
v.delete_needle(
&mut Needle {
id: NeedleId(*id),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(*id),
..Needle::default()
})
.unwrap();
}
Op::Empty(id) => {
-118
View File
@@ -110,7 +110,6 @@ fn build_test_state(
vacuum_commit_slots: None,
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
@@ -1436,120 +1435,3 @@ async fn write_queue_upload_returns_same_etag_as_direct_write() {
}
assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
}
// ============================================================================
// A Range over a compressed needle is answered from the decoded bytes
// ============================================================================
#[tokio::test]
async fn range_with_accept_gzip_on_compressed_needle_serves_decoded_bytes() {
let (state, _tmp) = test_state();
let uri = "/1/03637037d6/notes.txt";
let plain: Vec<u8> = (0..400)
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
.collect();
let app = build_admin_router(state.clone());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::from(plain.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let read = |range: Option<&str>| {
let mut request = Request::builder()
.uri(uri)
.header("Accept-Encoding", "gzip");
if let Some(range) = range {
request = request.header("Range", range);
}
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
};
// The whole read passes the stored gzip through, so its length is the compressed size.
let response = read(None).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()["Content-Encoding"], "gzip");
let compressed_len = body_bytes(response).await.len();
assert!(compressed_len < plain.len() / 4);
for (start, end) in [(10, 69), (compressed_len + 10, compressed_len + 69)] {
let response = read(Some(&format!("bytes={start}-{end}"))).await.unwrap();
assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
assert!(response.headers().get("Content-Encoding").is_none());
assert_eq!(
response.headers()["Content-Range"],
format!("bytes {start}-{end}/{}", plain.len()).as_str()
);
assert_eq!(body_bytes(response).await, &plain[start..=end]);
}
}
// ============================================================================
// HEAD of a compressed needle reports the length the matching GET would send
// ============================================================================
#[tokio::test]
async fn head_on_compressed_needle_reports_served_length() {
let (state, _tmp) = test_state();
let uri = "/1/04637037d6/notes.txt";
let plain: Vec<u8> = (0..400)
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
.collect();
let app = build_admin_router(state.clone());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::from(plain.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let send = |method: &str, accept_gzip: bool, range: Option<&str>| {
let mut request = Request::builder().method(method).uri(uri);
if accept_gzip {
request = request.header("Accept-Encoding", "gzip");
}
if let Some(range) = range {
request = request.header("Range", range);
}
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
};
let response = send("GET", true, None).await.unwrap();
let compressed_len = body_bytes(response).await.len();
assert!(compressed_len < plain.len() / 4);
for (accept_gzip, range, encoding, length) in [
(false, None, None, plain.len()),
(true, None, Some("gzip"), compressed_len),
(true, Some("bytes=0-9"), None, plain.len()),
] {
let response = send("HEAD", accept_gzip, range).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get("Content-Encoding")
.map(|v| v.to_str().unwrap()),
encoding,
"accept_gzip={accept_gzip} range={range:?}"
);
assert_eq!(
response.headers()["Content-Length"],
length.to_string().as_str(),
"accept_gzip={accept_gzip} range={range:?}"
);
}
}
-2
View File
@@ -7170,10 +7170,8 @@ dependencies = [
"hmac 0.12.1",
"http 1.5.0",
"hyper-util",
"jsonb",
"jsonwebtoken",
"lance",
"lance-arrow",
"lance-datafusion",
"lance-index",
"lance-linalg",
+1 -5
View File
@@ -35,12 +35,8 @@ lance-table = "10"
# For the execution options the sort needs; the scanner's own stream helper
# takes the default, which has spilling off.
lance-datafusion = "10"
arrow-array = "58"
arrow-schema = "58"
arrow-cast = "58"
# Lance's stored form of the metadata JSON, read when deriving the
# indexed-key columns.
jsonb = { version = "0.5", default-features = false }
chrono = "0.4"
futures = "0.3"
anyhow.workspace = true
@@ -92,7 +88,7 @@ protoc-bin-vendored = "3"
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tonic.workspace = true
lance-arrow = "10"
arrow-array = "58"
arrow-schema = "58"
arrow-cast = "58"
lance-linalg = "10"
+30 -163
View File
@@ -3,12 +3,11 @@ use std::collections::HashMap;
use anyhow::{Context, Result, anyhow};
use async_trait::async_trait;
use chrono::{Duration, Utc};
use lance::Dataset;
use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions};
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
use seaweed_worker_core::pb::{
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
RunDetectionRequest,
};
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
@@ -16,7 +15,6 @@ use tracing::warn;
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::purge::{self, Retention};
use crate::jobs::{clamp, string_list, table_id};
pub const JOB_TYPE: &str = "lance_cleanup_versions";
@@ -30,9 +28,7 @@ const MAX_MIN_VERSIONS: i64 = 1000;
/// Lance keeps every version until something removes it. Lance can also do this
/// itself through auto-cleanup, so this job is for deployments that would rather
/// the cluster owned the policy than each writer. A table may override the
/// retention in its own config, and may ask for a purge (see [`purge`]), which
/// this job carries out ahead of the retention window.
/// the cluster owned the policy than each writer.
pub struct CleanupVersionsHandler {
namespace_url: String,
storage: dataset::Storage,
@@ -72,8 +68,9 @@ impl CleanupVersionsHandler {
/// The oldest version that may be removed while still leaving `min_versions`
/// behind, or None when the table has no more than the floor.
async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<u64>> {
let mut versions: Vec<u64> = dataset
async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result<Option<u64>> {
let mut versions: Vec<u64> = table
.dataset
.versions()
.await?
.iter()
@@ -86,33 +83,6 @@ async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<
Ok(Some(versions[versions.len() - min_versions]))
}
/// The cleanup that keeps `min_versions` and every version younger than
/// `retain_hours`, or None when the floor leaves nothing to remove. A purge
/// cleans up through it too.
pub async fn retention_policy(
dataset: &Dataset,
retain_hours: i64,
min_versions: usize,
) -> Result<Option<CleanupPolicy>> {
let Some(floor) = version_floor(dataset, min_versions).await? else {
return Ok(None);
};
Ok(Some(CleanupPolicy {
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
// should_clean() ANDs its clauses, so a version has to be both older
// than the window and below the floor to go.
before_version: Some(floor),
// Files this dataset cannot account for are left alone: they may
// belong to a writer that has not committed yet, and deleting them
// would corrupt a commit in flight.
delete_unverified: false,
// A tagged version is pinned on purpose, so refuse rather than
// silently dropping what someone named.
error_if_tagged_old_versions: true,
..Default::default()
}))
}
#[async_trait]
impl JobHandler for CleanupVersionsHandler {
fn capability(&self) -> JobTypeCapability {
@@ -164,12 +134,6 @@ impl JobHandler for CleanupVersionsHandler {
defaults.clone(),
)),
worker_default_values: defaults,
// A purge compacts and retrains, so it gets a build's time.
admin_runtime_defaults: Some(AdminRuntimeDefaults {
job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
..Default::default()
}),
..Default::default()
}
}
@@ -217,23 +181,10 @@ impl JobHandler for CleanupVersionsHandler {
continue;
}
};
let config = table.dataset.config();
let pending_purge = purge::pending(config);
let floor = Retention::for_table(
config,
Retention {
retain_hours: DEFAULT_RETAIN_HOURS,
min_versions,
},
MAX_RETAIN_HOURS,
MAX_MIN_VERSIONS,
)
.min_versions;
// Age is decided at execution against the retention window; a table
// at or under the floor cannot lose a version whatever its age, so
// proposing one would only produce a job with nothing to do. A
// requested purge ignores the floor: it keeps only the latest.
if pending_purge.is_none() && stats.total_versions <= floor {
// proposing one would only produce a job with nothing to do.
if stats.total_versions <= min_versions {
continue;
}
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
@@ -242,21 +193,12 @@ impl JobHandler for CleanupVersionsHandler {
proposal_id: format!("{JOB_TYPE}:{encoded}"),
dedupe_key: format!("{JOB_TYPE}:{encoded}"),
job_type: JOB_TYPE.to_string(),
summary: match &pending_purge {
Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions),
None => format!("Clean up {encoded} ({} versions)", stats.total_versions),
},
detail: match &pending_purge {
Some(token) => format!(
"purge requested at {token}: rewrite fragments with deletions \
and keep only the latest version"
),
None => format!(
"{} versions retained, above the {floor} floor; \
those outside the retention window can go",
stats.total_versions
),
},
summary: format!("Clean up {encoded} ({} versions)", stats.total_versions),
detail: format!(
"{} versions retained, above the {min_versions} floor; \
those outside the retention window can go",
stats.total_versions
),
parameters,
..Default::default()
});
@@ -312,33 +254,14 @@ impl JobHandler for CleanupVersionsHandler {
) as usize;
let client = self.client();
let mut table = dataset::open(&client, &id, &self.storage).await?;
let table = dataset::open(&client, &id, &self.storage).await?;
let before = table.stats().await?;
if let Some(token) = purge::pending(table.dataset.config()) {
return self
.execute_purge(request, sender, &job.job_id, &mut table, &token)
.await;
}
let Retention {
retain_hours,
min_versions,
} = Retention::for_table(
table.dataset.config(),
Retention {
retain_hours,
min_versions,
},
MAX_RETAIN_HOURS,
MAX_MIN_VERSIONS,
);
// The floor is a promise about how much history survives, so it has to
// be applied here and not only when the job was proposed: by the time it
// runs, versions may have aged past the retention window, and age alone
// would take the table below what the operator asked to keep.
let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await?
else {
let Some(floor) = version_floor(&table, min_versions).await? else {
sender.send_completed(JobCompleted {
request_id: request.request_id.clone(),
job_id: job.job_id.clone(),
@@ -365,6 +288,20 @@ impl JobHandler for CleanupVersionsHandler {
..Default::default()
})?;
let policy = CleanupPolicy {
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
// should_clean() ANDs its clauses, so a version has to be both older
// than the window and below the floor to go.
before_version: Some(floor),
// Files this dataset cannot account for are left alone: they may
// belong to a writer that has not committed yet, and deleting them
// would corrupt a commit in flight.
delete_unverified: false,
// A tagged version is pinned on purpose, so refuse rather than
// silently dropping what someone named.
error_if_tagged_old_versions: true,
..Default::default()
};
let stats = cleanup_old_versions(&table.dataset, policy)
.await
.with_context(|| format!("clean up versions of {}", table.location))?;
@@ -406,73 +343,3 @@ impl JobHandler for CleanupVersionsHandler {
Ok(())
}
}
impl CleanupVersionsHandler {
async fn execute_purge(
&self,
request: &ExecuteJobRequest,
sender: &dyn ExecutionSender,
job_id: &str,
table: &mut dataset::OpenTable,
token: &str,
) -> Result<()> {
let stats = purge::purge(&mut table.dataset, token)
.await
.with_context(|| format!("purge {}", table.location))?;
if let Some(counters) = &self.metrics {
counters.versions_removed.inc_by(stats.old_versions);
counters.bytes_reclaimed.inc_by(stats.bytes_removed);
}
let mut output: HashMap<String, ConfigValue> = HashMap::new();
output.insert(
"fragments_rewritten".to_string(),
int_value(stats.fragments_rewritten as i64),
);
output.insert(
"old_versions_removed".to_string(),
int_value(stats.old_versions as i64),
);
output.insert(
"bytes_removed".to_string(),
int_value(stats.bytes_removed as i64),
);
sender.send_completed(JobCompleted {
request_id: request.request_id.clone(),
job_id: job_id.to_string(),
job_type: JOB_TYPE.to_string(),
success: true,
result: Some(JobResult {
output_values: output,
summary: if stats.served {
format!(
"purged: rewrote {} fragments, removed {} versions and {} bytes",
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
)
} else {
format!(
"purge pending: rewrote {} fragments, removed {} versions and {} bytes; \
versions from before the rebuild are still retained",
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
)
},
..Default::default()
}),
..Default::default()
})?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_purge_gets_a_builds_execution_timeout() {
let descriptor = CleanupVersionsHandler::new(String::new()).descriptor();
let defaults = descriptor.admin_runtime_defaults.unwrap();
assert_eq!(defaults.execution_timeout_seconds, 3600);
assert_eq!(defaults.job_type_max_runtime_seconds, 3600);
assert!(!defaults.enabled);
}
}
@@ -9,7 +9,6 @@
pub mod cleanup;
pub mod compact;
pub mod indices;
pub mod purge;
pub mod sort;
pub mod vector_index;
@@ -1,328 +0,0 @@
//! Per-table retention for version cleanup, and purges: compaction that
//! rewrites every fragment with deletions followed by a cleanup that keeps
//! only the latest version, so the bytes of deleted rows leave storage now
//! rather than after the retention window.
//!
//! Both are set in the table's own Lance config, so any client that can write
//! the table can set them and they travel with it:
//!
//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions`
//! override the worker's retention for this table.
//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`]
//! writes the current time); a purge is pending while it differs from
//! `seaweedfs.purge.completed`, which the purge sets to the token it served.
//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use anyhow::{Context, Result, bail};
use chrono::Utc;
use lance::Dataset;
use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions};
use lance::dataset::optimize::{CompactionOptions, compact_files};
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
use lance_index::metrics::NoOpMetricsCollector;
use lance_index::optimize::OptimizeOptions;
use lance_index::{IndexType, is_system_index};
use lance_table::format::IndexMetadata;
use tracing::warn;
use crate::jobs::cleanup::retention_policy;
pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours";
pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions";
pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested";
pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed";
pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt";
/// The retention cleanup applies to one table: the worker's, unless the
/// table's config overrides a value with one inside the worker's range.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Retention {
pub retain_hours: i64,
pub min_versions: usize,
}
impl Retention {
pub fn for_table(
config: &HashMap<String, String>,
worker: Retention,
max_retain_hours: i64,
max_min_versions: i64,
) -> Retention {
let read = |key: &str, min: i64, max: i64| -> Option<i64> {
let raw = config.get(key)?;
match raw.trim().parse::<i64>() {
Ok(value) if (min..=max).contains(&value) => Some(value),
_ => {
warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}");
None
}
}
};
Retention {
retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours)
.unwrap_or(worker.retain_hours),
min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions)
.map(|v| v as usize)
.unwrap_or(worker.min_versions),
}
}
}
/// The request token of a purge the table is waiting for, if any.
pub fn pending(config: &HashMap<String, String>) -> Option<String> {
let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?;
if requested.is_empty() {
return None;
}
match config.get(PURGE_COMPLETED_KEY) {
Some(completed) if completed.trim() == requested => None,
_ => Some(requested.to_string()),
}
}
/// Records a purge request on the table and returns its token. The next
/// cleanup detection proposes the purge.
pub async fn request_purge(dataset: &mut Dataset) -> Result<String> {
let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
dataset
.update_config([(PURGE_REQUESTED_KEY, token.as_str())])
.await
.context("record purge request")?;
Ok(token)
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct PurgeStats {
pub fragments_rewritten: usize,
pub old_versions: u64,
pub bytes_removed: u64,
/// False while versions from before the rebuild remain.
pub served: bool,
}
/// Rewrites every fragment that has deleted rows and rebuilds the indices from
/// the rows that remain, then removes the versions from before that through
/// the cleanup job's retention policy with only the latest version kept, and
/// marks `token` served once none of them is left. The rebuild is needed
/// because compaction only remaps an index: an IVF_FLAT index keeps the
/// deleted vector's values in its partitions under a remapped row address. It
/// runs once per request, so a purge that has to wait for its cleanup (a
/// version pinned by a tag, a query lease or one someone named, fails it)
/// retries only the cleanup. Files no version references stay while Lance
/// cannot tell them from a writer's uncommitted ones, which is up to seven
/// days for an abandoned write.
pub async fn purge(dataset: &mut Dataset, token: &str) -> Result<PurgeStats> {
let fragments_rewritten = rebuild(dataset, token).await?;
let cleanup = match retention_policy(dataset, 0, 1).await? {
Some(policy) => cleanup_old_versions(dataset, policy)
.await
.context("remove versions older than the purge")?,
None => RemovalStats::default(),
};
let served = finish(dataset, token).await?;
Ok(PurgeStats {
fragments_rewritten,
old_versions: cleanup.old_versions,
bytes_removed: cleanup.bytes_removed,
served,
})
}
/// Rewrites the fragments with deletions and rebuilds the indices for
/// `token`, unless that was done already, and returns how many fragments it
/// rewrote.
pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result<usize> {
if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) {
return Ok(0);
}
let vector_indices = vector_index_names(dataset).await?;
let options = CompactionOptions {
materialize_deletions: true,
// Zero materializes every fragment with any deletion.
materialize_deletions_threshold: 0.0,
..Default::default()
};
let compaction = compact_files(dataset, options, None)
.await
.context("rewrite fragments with deletions")?;
dataset.checkout_latest().await?;
if !vector_indices.is_empty() {
let mut options = OptimizeOptions::retrain();
options.index_names = Some(vector_indices);
dataset
.optimize_indices(&options)
.await
.context("rebuild indices without the deleted rows")?;
}
rebuild_scalar_indices(dataset).await?;
dataset.checkout_latest().await?;
dataset
.update_config([(PURGE_REBUILT_KEY, token)])
.await
.context("record purge rebuild")?;
Ok(compaction.fragments_removed)
}
/// Marks `token` served when the oldest version left is one the rebuild for
/// it wrote, so no version from before the rebuild remains.
pub async fn finish(dataset: &mut Dataset, token: &str) -> Result<bool> {
let oldest = dataset
.versions()
.await?
.iter()
.map(|v| v.version)
.min()
.context("the table has no versions")?;
let oldest = dataset.checkout_version(oldest).await?;
if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) {
return Ok(false);
}
dataset
.update_config([(PURGE_COMPLETED_KEY, token)])
.await
.context("record purge completion")?;
Ok(true)
}
/// Names of the vector indices. A retrain builds each name over the whole
/// table, so a column whose index is sharded over several names is refused:
/// its shards would overlap.
async fn vector_index_names(dataset: &Dataset) -> Result<Vec<String>> {
let mut by_column: BTreeMap<Vec<i32>, BTreeSet<String>> = BTreeMap::new();
for index in dataset
.load_indices()
.await?
.iter()
.filter(|i| is_vector(i))
{
by_column
.entry(index.fields.clone())
.or_default()
.insert(index.name.clone());
}
let mut names = Vec::new();
for shards in by_column.into_values() {
if shards.len() > 1 {
bail!(
"the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table"
);
}
names.extend(shards);
}
Ok(names)
}
/// Builds every scalar index again from the rows that remain. Compaction only
/// remaps them: a bitmap index keeps a deleted key, and under stable row ids
/// no scalar index is rewritten at all.
async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> {
let indices = dataset.load_indices().await?;
let mut rebuilt = BTreeSet::new();
for index in indices.iter() {
if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) {
continue;
}
let [field] = index.fields[..] else {
bail!("index {} covers {} columns", index.name, index.fields.len());
};
let column = dataset.schema().field_path(field)?;
let params = dataset
.open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector)
.await?
.derive_index_params()?;
dataset
.create_index(
&[column.as_str()],
IndexType::Scalar,
Some(index.name.clone()),
&params,
true,
)
.await
.with_context(|| format!("rebuild index {}", index.name))?;
}
Ok(())
}
fn is_vector(index: &IndexMetadata) -> bool {
index
.index_details
.as_ref()
.is_some_and(|details| details.type_url.ends_with("VectorIndexDetails"))
}
#[cfg(test)]
mod tests {
use super::*;
fn config(pairs: &[(&str, &str)]) -> HashMap<String, String> {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
const WORKER: Retention = Retention {
retain_hours: 168,
min_versions: 5,
};
#[test]
fn table_retention_overrides_within_range() {
assert_eq!(
Retention::for_table(&config(&[]), WORKER, 8760, 1000),
WORKER
);
assert_eq!(
Retention::for_table(
&config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]),
WORKER,
8760,
1000
),
Retention {
retain_hours: 0,
min_versions: 1
}
);
for bad in ["-1", "8761", "x", ""] {
assert_eq!(
Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000)
.retain_hours,
168,
"{bad:?}"
);
}
assert_eq!(
Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000)
.min_versions,
5
);
}
#[test]
fn purge_is_pending_until_its_token_is_served() {
assert_eq!(pending(&config(&[])), None);
assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None);
assert_eq!(
pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])),
Some("t1".to_string())
);
assert_eq!(
pending(&config(&[
(PURGE_REQUESTED_KEY, "t2"),
(PURGE_COMPLETED_KEY, "t1")
])),
Some("t2".to_string())
);
assert_eq!(
pending(&config(&[
(PURGE_REQUESTED_KEY, "t2"),
(PURGE_COMPLETED_KEY, "t2")
])),
None
);
}
}
-1
View File
@@ -11,7 +11,6 @@ pub mod dataset;
pub mod direct_read;
pub mod grpc_tls;
pub mod jobs;
pub mod metadata_index;
pub mod metrics;
pub mod preview;
pub mod security;
@@ -1,285 +0,0 @@
//! Metadata keys an S3 Vectors index keeps a scalar index for.
//!
//! Each indexed key is a hidden `_mk_<n>` column of `List<Utf8>`: the strings
//! an S3 `$eq` on a string matches, that is the value when it is a string or
//! the string elements of an array, and nothing for any other type or a
//! missing key. The field metadata names the key and its state. The
//! `metadata` JSON column stays the source of truth: every writer derives the
//! columns from it, and queries use one only once its state is `ready`.
use std::sync::Arc;
use anyhow::{Result, bail};
use arrow_array::builder::{ListBuilder, StringBuilder};
use arrow_array::{Array, ArrayRef, LargeBinaryArray, StringArray};
use arrow_schema::{DataType, Field, Schema};
pub const COLUMN_PREFIX: &str = "_mk_";
/// Field metadata naming the metadata key a column holds.
pub const KEY_FIELD_META: &str = "weed:metadata_key";
/// Field metadata holding the column's [`State`].
pub const STATE_FIELD_META: &str = "weed:metadata_key_state";
/// Indexed keys an index may have.
pub const MAX_KEYS: usize = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum State {
/// The column exists; its scalar index may not.
Building,
/// The column and its scalar index exist; queries may use them.
Ready,
}
impl State {
pub fn as_str(self) -> &'static str {
match self {
State::Building => "building",
State::Ready => "ready",
}
}
pub fn parse(s: &str) -> Option<Self> {
match s {
"building" => Some(State::Building),
"ready" => Some(State::Ready),
_ => None,
}
}
}
/// One indexed key, as the table's schema records it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IndexedKey {
pub column: String,
pub key: String,
/// None for a state this version does not know, which no query uses.
pub state: Option<State>,
}
/// The type of every indexed-key column.
pub fn column_type() -> DataType {
DataType::List(Arc::new(Field::new("item", DataType::Utf8, true)))
}
/// The field of the column holding `key`.
pub fn field(column: &str, key: &str, state: State) -> Field {
Field::new(column, column_type(), true).with_metadata(
[
(KEY_FIELD_META.to_string(), key.to_string()),
(STATE_FIELD_META.to_string(), state.as_str().to_string()),
]
.into_iter()
.collect(),
)
}
/// The indexed-key columns of `schema`, in schema order.
pub fn indexed_keys(schema: &Schema) -> Vec<IndexedKey> {
schema
.fields()
.iter()
.filter_map(|f| indexed_key(f))
.collect()
}
/// The key `field` holds, when it is an indexed-key column.
pub fn indexed_key(field: &Field) -> Option<IndexedKey> {
if !field.name().starts_with(COLUMN_PREFIX) {
return None;
}
let key = field.metadata().get(KEY_FIELD_META)?;
Some(IndexedKey {
column: field.name().clone(),
key: key.clone(),
state: field
.metadata()
.get(STATE_FIELD_META)
.and_then(|s| State::parse(s)),
})
}
/// The strings an S3 `$eq` on a string matches for this stored value.
pub fn json_strings(value: Option<&serde_json::Value>) -> Vec<&str> {
match value {
Some(serde_json::Value::String(s)) => vec![s.as_str()],
Some(serde_json::Value::Array(items)) => items.iter().filter_map(|v| v.as_str()).collect(),
_ => Vec::new(),
}
}
/// [`json_strings`] for `key` of one row's metadata as Lance stores it (jsonb).
pub fn jsonb_strings(metadata: &[u8], key: &str) -> Result<Vec<String>> {
let raw = jsonb::RawJsonb::new(metadata);
let Some(value) = raw.get_by_name(key, false)? else {
return Ok(Vec::new());
};
let value = value.as_raw();
if let Some(s) = value.as_str()? {
return Ok(vec![s.into_owned()]);
}
let Some(items) = value.array_values()? else {
return Ok(Vec::new());
};
let mut out = Vec::new();
for item in items {
if let Some(s) = item.as_raw().as_str()? {
out.push(s.into_owned());
}
}
Ok(out)
}
/// The column for `key` over rows whose metadata is given as JSON objects.
pub fn column_from_json<'a>(
rows: impl IntoIterator<Item = Option<&'a serde_json::Value>>,
key: &str,
) -> ArrayRef {
let mut builder = ListBuilder::new(StringBuilder::new());
for row in rows {
for s in json_strings(row.and_then(|m| m.get(key))) {
builder.values().append_value(s);
}
builder.append(true);
}
Arc::new(builder.finish())
}
/// The column for `key` over a stored `metadata` column: jsonb (LargeBinary)
/// or JSON text (Utf8).
pub fn column_from_metadata(metadata: &dyn Array, key: &str) -> Result<ArrayRef> {
let mut builder = ListBuilder::new(StringBuilder::new());
if let Some(jsonb) = metadata.as_any().downcast_ref::<LargeBinaryArray>() {
for i in 0..jsonb.len() {
if !jsonb.is_null(i) {
for s in jsonb_strings(jsonb.value(i), key)? {
builder.values().append_value(s);
}
}
builder.append(true);
}
} else if let Some(text) = metadata.as_any().downcast_ref::<StringArray>() {
for i in 0..text.len() {
if !text.is_null(i) {
let row: serde_json::Value = serde_json::from_str(text.value(i))?;
for s in json_strings(row.get(key)) {
builder.values().append_value(s);
}
}
builder.append(true);
}
} else {
bail!("metadata column is {}", metadata.data_type());
}
Ok(Arc::new(builder.finish()))
}
#[cfg(test)]
mod tests {
use super::*;
use arrow_array::ListArray;
fn lists(array: &ArrayRef) -> Vec<Vec<String>> {
let list = array.as_any().downcast_ref::<ListArray>().unwrap();
(0..list.len())
.map(|i| {
assert!(!list.is_null(i));
let values = list.value(i);
let values = values.as_any().downcast_ref::<StringArray>().unwrap();
values.iter().map(|v| v.unwrap().to_string()).collect()
})
.collect()
}
const ROWS: [Option<&str>; 12] = [
Some(r#"{"k": "a"}"#),
Some(r#"{"k": 7}"#),
Some(r#"{"k": "7"}"#),
Some(r#"{"k": ["a", "b", 7, true, null, ["c"], {"d": "e"}]}"#),
Some(r#"{"x": "a"}"#),
Some(r#"{"k": null}"#),
Some(r#"{"k": true}"#),
Some(r#"{"k": {"a": "b"}}"#),
Some(r#"{"k": "line\nbreak\u0001"}"#),
Some(r#"{"k": []}"#),
Some(r#"{"K": "upper"}"#),
None,
];
fn want() -> Vec<Vec<String>> {
let s = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::<Vec<_>>();
vec![
s(&["a"]),
s(&[]),
s(&["7"]),
s(&["a", "b"]),
s(&[]),
s(&[]),
s(&[]),
s(&[]),
s(&["line\nbreak\u{1}"]),
s(&[]),
s(&[]),
s(&[]),
]
}
#[test]
fn json_and_jsonb_rows_derive_the_same_strings() {
let values: Vec<Option<serde_json::Value>> = ROWS
.iter()
.map(|r| r.map(|r| serde_json::from_str(r).unwrap()))
.collect();
assert_eq!(
lists(&column_from_json(values.iter().map(|v| v.as_ref()), "k")),
want()
);
let text = StringArray::from(ROWS.to_vec());
assert_eq!(lists(&column_from_metadata(&text, "k").unwrap()), want());
let jsonb = lance_arrow::json::JsonArray::try_from_iter(ROWS.iter().copied())
.unwrap()
.into_inner();
assert_eq!(lists(&column_from_metadata(&jsonb, "k").unwrap()), want());
}
#[test]
fn indexed_keys_come_from_the_field_metadata() {
let mut unknown = field("_mk_2", "c", State::Ready);
unknown.set_metadata(
[
(KEY_FIELD_META.to_string(), "c".to_string()),
(STATE_FIELD_META.to_string(), "later".to_string()),
]
.into_iter()
.collect(),
);
let schema = Schema::new(vec![
Field::new("key", DataType::Utf8, false),
field("_mk_0", "a.b", State::Ready),
Field::new("_mk_1", column_type(), true),
field("_mk_3", "d", State::Building),
unknown,
]);
assert_eq!(
indexed_keys(&schema),
vec![
IndexedKey {
column: "_mk_0".into(),
key: "a.b".into(),
state: Some(State::Ready),
},
IndexedKey {
column: "_mk_3".into(),
key: "d".into(),
state: Some(State::Building),
},
IndexedKey {
column: "_mk_2".into(),
key: "c".into(),
state: None,
},
]
);
}
}
-394
View File
@@ -1,394 +0,0 @@
//! A purge removes a deleted vector's bytes from storage: after it, no file
//! under the table holds the vector, its key or its metadata. Runs on a local
//! table, so it needs no cluster.
use std::path::Path;
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use lance::Dataset;
use lance::dataset::cleanup::{CleanupPolicy, RemovalStats, cleanup_old_versions};
use lance::index::DatasetIndexExt;
use lance::index::vector::VectorIndexParams;
use lance_index::IndexType;
use lance_linalg::distance::DistanceType;
use weed_lance_worker::jobs::purge;
const DIM: i32 = 8;
const ROWS: usize = 300;
const MARKER_ROW: usize = 137;
const KEY_MARKER: &str = "purge-key-5f0c2e9a7d41b3c8";
const METADATA_MARKER: &str = "purge-metadata-a93e1d5b77c2f046";
fn marker_vector() -> Vec<f32> {
// Values no other row has, so their bytes identify the row.
(0..DIM).map(|i| 31_337.125 + i as f32 * 0.5).collect()
}
fn batch() -> (Arc<Schema>, RecordBatch) {
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
Field::new("metadata", DataType::Utf8, true),
]));
let keys: Vec<String> = (0..ROWS)
.map(|i| {
if i == MARKER_ROW {
KEY_MARKER.to_string()
} else {
format!("k{i}")
}
})
.collect();
let metadata: Vec<String> = (0..ROWS)
.map(|i| {
if i == MARKER_ROW {
format!("{{\"note\":\"{METADATA_MARKER}\"}}")
} else {
format!("{{\"n\":{i}}}")
}
})
.collect();
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..ROWS).map(|i| {
let values: Vec<f32> = if i == MARKER_ROW {
marker_vector()
} else {
(0..DIM)
.map(|d| ((i * 7 + d as usize) % 97) as f32)
.collect()
};
Some(values.into_iter().map(Some).collect::<Vec<_>>())
}),
DIM,
);
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(StringArray::from(keys)),
Arc::new(vectors),
Arc::new(StringArray::from(metadata)),
],
)
.unwrap();
(schema, batch)
}
fn vector_bytes() -> Vec<u8> {
marker_vector()
.iter()
.flat_map(|v| v.to_le_bytes())
.collect()
}
/// Each marker found under `dir`, as "<marker> in <file>".
fn markers_in(dir: &Path) -> Vec<String> {
let vector = vector_bytes();
let mut found = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(path) = stack.pop() {
for entry in std::fs::read_dir(&path).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
stack.push(entry.path());
continue;
}
let bytes = std::fs::read(entry.path()).unwrap();
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
for (name, needle) in [
("vector", vector.as_slice()),
("key", KEY_MARKER.as_bytes()),
("metadata", METADATA_MARKER.as_bytes()),
] {
if has(needle) {
let file = entry.path();
let file = file.strip_prefix(dir).unwrap_or(&file);
found.push(format!("{name} in {}", file.display()));
}
}
}
}
found.sort_unstable();
found
}
/// A purge whose cleanup nothing holds back, so only the latest version
/// stays. [`purge::purge`] cleans up through the cleanup job's retention
/// policy, which a deployment may have keep more.
async fn purge_to_latest(
dataset: &mut Dataset,
token: &str,
) -> anyhow::Result<(usize, RemovalStats, bool)> {
let rewritten = purge::rebuild(dataset, token).await?;
let policy = CleanupPolicy {
before_timestamp: Some(chrono::Utc::now()),
before_version: Some(dataset.latest_version_id().await?),
error_if_tagged_old_versions: true,
..Default::default()
};
let removed = cleanup_old_versions(dataset, policy).await?;
Ok((rewritten, removed, purge::finish(dataset, token).await?))
}
#[tokio::test]
async fn purge_removes_a_deleted_vectors_bytes() {
purge_leaves_no_trace(false).await;
}
/// Under stable row ids compaction leaves scalar indices as they are, so only
/// a rebuild takes the key out of them.
#[tokio::test]
async fn purge_removes_a_deleted_vectors_bytes_under_stable_row_ids() {
purge_leaves_no_trace(true).await;
}
async fn purge_leaves_no_trace(stable_row_ids: bool) {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().join("index.lance");
let uri = uri.to_str().unwrap();
let (schema, batch) = batch();
let params = lance::dataset::WriteParams {
enable_stable_row_ids: stable_row_ids,
..Default::default()
};
let mut dataset = Dataset::write(
RecordBatchIterator::new(vec![Ok(batch)], schema),
uri,
Some(params),
)
.await
.unwrap();
dataset
.create_index(
&["vector"],
IndexType::Vector,
Some("vector_idx".to_string()),
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
false,
)
.await
.unwrap();
dataset
.create_index(
&["key"],
IndexType::BTree,
Some("key_idx".to_string()),
&lance_index::scalar::ScalarIndexParams::default(),
false,
)
.await
.unwrap();
// A remapped bitmap index still lists the deleted row's value.
dataset
.create_index(
&["metadata"],
IndexType::Bitmap,
Some("metadata_idx".to_string()),
&lance_index::scalar::ScalarIndexParams::for_builtin(
lance_index::scalar::BuiltinIndexType::Bitmap,
),
false,
)
.await
.unwrap();
dataset.checkout_latest().await.unwrap();
let before = markers_in(dir.path());
assert!(
before.iter().any(|m| m.starts_with("vector ")),
"the vector's bytes should be found before deletion, found {before:?}"
);
dataset
.delete(&format!("key = '{KEY_MARKER}'"))
.await
.unwrap();
assert_eq!(dataset.count_rows(None).await.unwrap(), ROWS - 1);
// A deletion alone, and the default retention cleanup, leave the bytes:
// the fragment still holds the row and older versions still reference it.
cleanup_old_versions(
&dataset,
CleanupPolicy {
before_timestamp: Some(chrono::Utc::now() - chrono::Duration::hours(168)),
..Default::default()
},
)
.await
.unwrap();
assert!(
markers_in(dir.path())
.iter()
.any(|m| m.starts_with("vector ")),
"a deleted vector is expected to stay until purged"
);
let token = purge::request_purge(&mut dataset).await.unwrap();
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
assert!(served);
assert!(rewritten >= 1);
assert!(removed.old_versions >= 1, "{removed:?}");
assert_eq!(
markers_in(dir.path()),
Vec::<String>::new(),
"deleted vector's bytes remain in storage after the purge"
);
assert_eq!(purge::pending(dataset.config()), None);
// The table is intact: every other row, and the index, still serve.
let reopened = Dataset::open(uri).await.unwrap();
assert_eq!(reopened.count_rows(None).await.unwrap(), ROWS - 1);
assert_eq!(purge::pending(reopened.config()), None);
assert_eq!(reopened.load_indices().await.unwrap().len(), 3);
}
/// A version a query lease (a tag) pins still holds the deleted row, so the
/// purge must fail and leave the request pending rather than report success.
/// Once the tag goes, the next purge only cleans up: the rebuild is not done
/// again.
#[tokio::test]
async fn purge_refuses_while_a_tag_pins_old_data() {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().join("index.lance");
let uri = uri.to_str().unwrap();
let (schema, batch) = batch();
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
.await
.unwrap();
dataset
.create_index(
&["vector"],
IndexType::Vector,
Some("vector_idx".to_string()),
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
false,
)
.await
.unwrap();
let pinned = dataset.version().version;
dataset
.tags()
.create(
"weed-vector-query-lease",
lance::dataset::refs::Ref::Version(None, Some(pinned)),
)
.await
.unwrap();
dataset
.delete(&format!("key = '{KEY_MARKER}'"))
.await
.unwrap();
let token = purge::request_purge(&mut dataset).await.unwrap();
assert!(purge_to_latest(&mut dataset, &token).await.is_err());
dataset.checkout_latest().await.unwrap();
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
let index = |dataset: &Dataset| {
let dataset = dataset.clone();
async move { dataset.load_indices().await.unwrap()[0].uuid }
};
let rebuilt = index(&dataset).await;
dataset
.tags()
.delete("weed-vector-query-lease")
.await
.unwrap();
let (rewritten, _, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
assert!(served);
assert_eq!(rewritten, 0);
assert_eq!(
index(&dataset).await,
rebuilt,
"the index was rebuilt again"
);
assert_eq!(purge::pending(dataset.config()), None);
assert_eq!(markers_in(dir.path()), Vec::<String>::new());
}
/// The request is served only once no version from before the rebuild is
/// left, whatever kept one.
#[tokio::test]
async fn purge_is_served_once_older_versions_are_gone() {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().join("index.lance");
let uri = uri.to_str().unwrap();
let (schema, batch) = batch();
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
.await
.unwrap();
dataset
.delete(&format!("key = '{KEY_MARKER}'"))
.await
.unwrap();
let token = purge::request_purge(&mut dataset).await.unwrap();
assert_eq!(purge::rebuild(&mut dataset, &token).await.unwrap(), 1);
assert!(!purge::finish(&mut dataset, &token).await.unwrap());
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
assert!(served);
assert_eq!(rewritten, 0);
assert!(removed.old_versions >= 1, "{removed:?}");
assert_eq!(purge::pending(dataset.config()), None);
}
/// A retrain builds every index name over the whole table, so shards of one
/// vector index would end up overlapping: the purge refuses before touching
/// the table and leaves the request pending.
#[tokio::test]
async fn purge_refuses_a_sharded_vector_index() {
let dir = tempfile::tempdir().unwrap();
let uri = dir.path().join("index.lance");
let uri = uri.to_str().unwrap();
let (schema, batch) = batch();
let params = lance::dataset::WriteParams {
max_rows_per_file: ROWS / 2,
..Default::default()
};
let mut dataset = Dataset::write(
RecordBatchIterator::new(vec![Ok(batch)], schema),
uri,
Some(params),
)
.await
.unwrap();
let fragments: Vec<u32> = dataset
.get_fragments()
.iter()
.map(|f| f.id() as u32)
.collect();
assert_eq!(fragments.len(), 2);
for (shard, fragment) in fragments.iter().enumerate() {
dataset
.create_index_builder(
&["vector"],
IndexType::Vector,
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
)
.name(format!("shard-{shard}"))
.fragments(vec![*fragment])
.await
.unwrap();
dataset.checkout_latest().await.unwrap();
}
dataset
.delete(&format!("key = '{KEY_MARKER}'"))
.await
.unwrap();
let token = purge::request_purge(&mut dataset).await.unwrap();
let version = dataset.version().version;
let err = purge::purge(&mut dataset, &token).await.unwrap_err();
assert!(format!("{err:#}").contains("sharded"), "{err:#}");
dataset.checkout_latest().await.unwrap();
assert_eq!(dataset.version().version, version, "the table was changed");
assert_eq!(purge::pending(dataset.config()), Some(token));
}
+3 -3
View File
@@ -13,9 +13,9 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/common v0.71.0 // indirect
github.com/prometheus/procfs v0.23.0 // indirect
golang.org/x/sys v0.49.0 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
replace github.com/seaweedfs/seaweedfs => ../..
+16 -12
View File
@@ -2,31 +2,35 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
github.com/prometheus/procfs v0.23.0 h1:ad+S6FTkzwaMgacKLM5h6bNKExS+DMH0YiUwXItH/o4=
github.com/prometheus/procfs v0.23.0/go.mod h1:3AVXO0FqLafyYFU/V/IQRZlC8Rgqz8Dm/x6pKLK+RIk=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+1 -22
View File
@@ -74,15 +74,9 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
if limit > 0 && len(series.Versions) > limit {
// Old releases are a long tail of one-cluster bands; keep the versions
// most of the fleet is on now and sum the rest into one band. The
// newest release is kept too, while it is still too small to rank, or
// clusters upgrading to it look like they left the named versions.
newest := newestRelease(series.Versions, last)
// most of the fleet is on now and sum the rest into one band.
ranked := append([]VersionCounts(nil), series.Versions...)
sort.Slice(ranked, func(i, j int) bool {
if (ranked[i].Version == newest) != (ranked[j].Version == newest) {
return ranked[i].Version == newest
}
if ranked[i].Clusters[last] != ranked[j].Clusters[last] {
return ranked[i].Clusters[last] > ranked[j].Clusters[last]
}
@@ -110,21 +104,6 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
return series
}
// newestRelease is the newest plain release, without a build suffix, that
// clusters run on the last day.
func newestRelease(versions []VersionCounts, last int) string {
newest := ""
for _, v := range versions {
if _, suffix, ok := versionParts(v.Version); !ok || suffix != "" || v.Clusters[last] == 0 {
continue
}
if newest == "" || versionLess(newest, v.Version) {
newest = v.Version
}
}
return newest
}
// versionedHistories drops the samples recorded before the reported version was
// kept in history, so the version series spans the days it actually knows a
// version for instead of climbing out of a run of blank days.
-26
View File
@@ -93,32 +93,6 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
}
}
func TestVersionSeriesKeepsNewestRelease(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "a", HistorySample{Version: "4.40"}, -2, -1, 0)
seedSamples(s, "b", HistorySample{Version: "4.40"}, -2, -1, 0)
seedSamples(s, "c", HistorySample{Version: "4.41"}, -2, -1, 0)
seedSamples(s, "d", HistorySample{Version: "4.41"}, -2, -1, 0)
// Upgraded to the newest release, which is still the smallest band.
seedSamples(s, "e", HistorySample{Version: "4.41"}, -2)
seedSamples(s, "e", HistorySample{Version: "4.42"}, -1, 0)
// A suffixed build newer than every release does not take its place.
seedSamples(s, "f", HistorySample{Version: "4.43-dev"}, -2, -1, 0)
series := s.GetVersionSeries(3, 2)
var kept []string
for _, v := range series.Versions {
kept = append(kept, v.Version)
}
if len(kept) != 2 || kept[0] != "4.41" || kept[1] != "4.42" {
t.Fatalf("kept = %v, want 4.41 and the newest release 4.42", kept)
}
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 4.40 and 4.43-dev", series.Other)
}
}
func TestVersionLess(t *testing.T) {
// Ordered oldest to newest; every pair must compare in this order.
ordered := []string{"unknown", "3.97", "4.02", "4.30", "4.30-enterprise", "9.99", "10.02"}
+8 -19
View File
@@ -28,13 +28,11 @@ import (
const filerGroup = "fuse-dlm-test"
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
// mounts for testing DLM-based cross-mount write coordination, and what two
// mounts do without it.
// mounts for testing DLM-based cross-mount write coordination.
type dlmTestCluster struct {
t testing.TB
baseDir string
weedBinary string
dlm bool // the mounts run with -dlm
masterPort int
masterGrpcPort int
@@ -54,11 +52,6 @@ type dlmTestCluster struct {
}
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
return startTestCluster(t, true)
}
// startTestCluster starts the cluster; dlm says whether the mounts run with -dlm.
func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
binary := findWeedBinary()
if binary == "" {
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
@@ -71,7 +64,6 @@ func startTestCluster(t testing.TB, dlm bool) *dlmTestCluster {
t: t,
baseDir: baseDir,
weedBinary: binary,
dlm: dlm,
}
// Register cleanup early so processes are stopped even if a require fails below.
t.Cleanup(c.Stop)
@@ -232,20 +224,17 @@ func (c *dlmTestCluster) startMount(idx int, configDir string) error {
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return fmt.Errorf("create cache dir: %w", err)
}
args := []string{
"-logdir=" + filepath.Join(c.baseDir, "logs"),
c.mountCmds[idx] = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"mount",
"-filer=" + c.filerAddress(0), // both mounts use filer0 for shared metadata
"-dir=" + c.mountPoints[idx],
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
"-dir="+c.mountPoints[idx],
"-filer.path=/",
"-dirAutoCreate",
"-allowOthers=false",
"-cacheDir=" + cacheDir,
}
if c.dlm {
args = append(args, "-dlm")
}
c.mountCmds[idx] = exec.Command(c.weedBinary, args...)
"-cacheDir="+cacheDir,
"-dlm",
)
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
}
-177
View File
@@ -1,177 +0,0 @@
//go:build linux || darwin
package fuse_dlm
import (
"bytes"
"context"
"fmt"
"io"
"math/rand"
"net/http"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// TestStageSameFilesWithoutDLM has two mounts without -dlm stage the same
// files into one directory at the same time, the way two jobs extract the same
// archive into a shared directory: a file that exists is skipped, otherwise it
// is created, written and closed, then chmod'ed and given its mtime. Both
// mounts often create the same file, and each then changes the attributes of
// the copy it has cached.
//
// Whatever either mount is told, no entry may point at a chunk that is gone
// from the volume server, and every file must read back whole. An update sent
// from a cached copy after the other mount rewrote the file puts back the
// chunks the rewrite dropped, which the filer deletes; reads of such a file
// fail once the volume is vacuumed.
func TestStageSameFilesWithoutDLM(t *testing.T) {
if testing.Short() {
t.Skip("skipping FUSE integration test in short mode")
}
cluster := startTestCluster(t, false)
t.Cleanup(cluster.Stop)
const (
files = 400
dir = "staged"
)
rng := rand.New(rand.NewSource(1))
names := make([]string, files)
content := make(map[string][]byte, files)
for i := range names {
names[i] = fmt.Sprintf("img-%04d.jpg", i)
data := make([]byte, 1024+rng.Intn(63*1024))
rng.Read(data)
content[names[i]] = data
}
mtime := time.Date(2026, 10, 26, 4, 0, 0, 0, time.UTC)
// What the losing mount is told depends on the version (nothing, EIO,
// EAGAIN); the files are what this test checks.
failed := make([][]string, 2)
setupErr := make([]error, 2)
wrote := [2][]bool{make([]bool, files), make([]bool, files)}
var ready, wg sync.WaitGroup
ready.Add(2)
for m := 0; m < 2; m++ {
wg.Add(1)
go func(m int) {
defer wg.Done()
stage := filepath.Join(cluster.mountPoints[m], dir)
setupErr[m] = os.MkdirAll(stage, 0755)
ready.Done()
ready.Wait()
if setupErr[m] != nil {
return
}
for i, name := range names {
p := filepath.Join(stage, name)
if _, err := os.Lstat(p); err == nil {
continue
}
wrote[m][i] = true
if err := os.WriteFile(p, content[name], 0644); err != nil {
failed[m] = append(failed[m], err.Error())
continue
}
if err := os.Chmod(p, 0640); err != nil {
failed[m] = append(failed[m], err.Error())
continue
}
if err := os.Chtimes(p, mtime, mtime); err != nil {
failed[m] = append(failed[m], err.Error())
}
}
}(m)
}
wg.Wait()
for m, f := range failed {
require.NoError(t, setupErr[m], "mount%d", m)
if len(f) > 0 {
t.Logf("mount%d: %d calls failed, first: %s", m, len(f), f[0])
}
}
both := 0
for i := range names {
if wrote[0][i] && wrote[1][i] {
both++
}
}
t.Logf("both mounts wrote %d of %d files", both, files)
require.Positive(t, both, "the mounts never wrote the same file")
// The filer deletes the chunks an update dropped shortly after the update.
time.Sleep(5 * time.Second)
conn, err := grpc.NewClient(cluster.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
require.NoError(t, err)
defer conn.Close()
client := filer_pb.NewSeaweedFilerClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
var broken []string
for _, name := range names {
if ctx.Err() != nil {
broken = append(broken, "checks did not finish: "+ctx.Err().Error())
break
}
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/" + dir,
Name: name,
})
if err != nil {
broken = append(broken, fmt.Sprintf("%s: lookup: %v", name, err))
continue
}
for _, chunk := range resp.Entry.GetChunks() {
if !cluster.needleExists(ctx, t, chunk.GetFileIdString()) {
broken = append(broken, fmt.Sprintf("%s: chunk %s is deleted", name, chunk.GetFileIdString()))
}
}
if got, err := cluster.readFromFiler(ctx, "/"+dir+"/"+name); err != nil {
broken = append(broken, fmt.Sprintf("%s: read: %v", name, err))
} else if !bytes.Equal(got, content[name]) {
broken = append(broken, fmt.Sprintf("%s: read %d bytes, want the %d written", name, len(got), len(content[name])))
}
}
require.Empty(t, broken, "%d of %d staged files are broken", len(broken), files)
}
// needleExists reports whether the volume server still has the needle; a
// deleted one answers 404 (only readDeleted=true would serve it).
func (c *dlmTestCluster) needleExists(ctx context.Context, t testing.TB, fid string) bool {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("http://127.0.0.1:%d/%s", c.volumePort, fid), nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
// readFromFiler reads a file whole through filer0's HTTP API.
func (c *dlmTestCluster) readFromFiler(ctx context.Context, path string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPorts[0], path), nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %s", resp.Status)
}
return io.ReadAll(resp.Body)
}
+1 -14
View File
@@ -29,7 +29,6 @@ type FuseTestFramework struct {
mountProcess *managedProcess
filerAddr string
filerPort int
masterPort int
weedBinary string
isSetup bool
config *TestConfig
@@ -75,10 +74,6 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
require.NoError(t, err)
filerPort := freePort(t)
masterPort := freePort(t)
for masterPort == filerPort {
masterPort = freePort(t)
}
return &FuseTestFramework{
t: t,
@@ -88,7 +83,6 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
logDir: filepath.Join(tempDir, "logs"),
filerPort: filerPort,
filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort),
masterPort: masterPort,
weedBinary: findWeedBinary(),
isSetup: false,
config: config,
@@ -224,11 +218,6 @@ func (f *FuseTestFramework) GetFilerAddr() string {
return f.filerAddr
}
// GetMasterAddr returns the master HTTP address of the mini cluster.
func (f *FuseTestFramework) GetMasterAddr() string {
return fmt.Sprintf("127.0.0.1:%d", f.masterPort)
}
// managedProcess is a started weed sub-command whose exit is watched, so a wait
// for it to come up ends the moment it dies instead of burning its full timeout.
type managedProcess struct {
@@ -332,7 +321,7 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
args = append(args, "-v=4")
}
adminPort := freePort(f.t)
for adminPort == f.filerPort || adminPort == f.masterPort {
for adminPort == f.filerPort {
adminPort = freePort(f.t)
}
args = append(args,
@@ -341,8 +330,6 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-filer.port="+strconv.Itoa(f.filerPort),
"-master.port="+strconv.Itoa(f.masterPort),
"-master.port.grpc="+strconv.Itoa(f.masterPort+grpcPortOffset),
"-s3=false",
"-webdav=false",
"-admin.ui=false",
@@ -1,214 +0,0 @@
//go:build linux
package fuse_test
import (
"encoding/binary"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
const (
mergePageSize = 8192
mergePages = 32
)
// mergeHTTP bounds every HTTP call the test makes to the cluster.
var mergeHTTP = &http.Client{Timeout: 30 * time.Second}
// mergeVacuumHTTP outlasts a master vacuum, which waits up to 30 s per volume
// for writes assigned to it to drain before compacting it.
var mergeVacuumHTTP = &http.Client{Timeout: 3 * time.Minute}
// TestReadOpenHandleAfterChunkMerge covers a file a flush merged while a
// descriptor kept it open. The merge rewrites the file into new chunks and the
// filer deletes the chunks it replaced, so after a vacuum only the new chunks
// exist. Reads through the open handle must find them at once, whether or not
// the handle has a write it has not flushed yet.
func TestReadOpenHandleAfterChunkMerge(t *testing.T) {
config := DefaultTestConfig()
// The client chunk cache could still hold a replaced chunk; the reads
// below have to reach the volume server.
config.CacheSizeMB = 0
config.MountOptions = append(config.MountOptions, "-cacheCapacityMB=0")
framework := NewFuseTestFramework(t, config)
defer framework.Cleanup()
require.NoError(t, framework.Setup(config))
t.Run("CleanHandle", func(t *testing.T) {
testReadOpenHandleAfterChunkMerge(t, framework, "merged_clean.dat", false)
})
t.Run("HandleWithUnflushedWrite", func(t *testing.T) {
testReadOpenHandleAfterChunkMerge(t, framework, "merged_dirty.dat", true)
})
}
func testReadOpenHandleAfterChunkMerge(t *testing.T, framework *FuseTestFramework, name string, unflushedWrite bool) {
path := filepath.Join(framework.GetMountPoint(), name)
held, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
require.NoError(t, err)
defer held.Close()
version := make([]int, mergePages)
whole := make([]byte, 0, mergePages*mergePageSize)
for p := 0; p < mergePages; p++ {
whole = append(whole, mergePagePattern(p, 0)...)
}
_, err = held.WriteAt(whole, 0)
require.NoError(t, err)
require.NoError(t, held.Sync())
// Overlapping 16-page windows: each leaves part of every older window
// visible, so the stored chunks soon add up to more than twice the file
// size and the next flush merges them.
var replaced []mergeChunk
for i, start := range []int{0, 3, 6, 9, 12, 15} {
before := mergeEntryChunks(t, framework, name)
window := make([]byte, 0, 16*mergePageSize)
for p := start; p < start+16; p++ {
window = append(window, mergePagePattern(p, i+1)...)
version[p] = i + 1
}
_, err = held.WriteAt(window, int64(start*mergePageSize))
require.NoError(t, err)
require.NoError(t, held.Sync())
if after := mergeEntryChunks(t, framework, name); len(after) == 1 && after[0].ModifiedTsNs == 0 {
replaced = before
break
}
}
require.NotEmpty(t, replaced, "the overlapping writes never made a flush merge the chunks")
// The filer deletes the replaced chunks after the merge commits; vacuum
// once they are deleted so that only the merged chunk is left on disk.
master := framework.GetMasterAddr()
require.Eventually(t, func() bool {
for _, c := range replaced {
if mergeChunkStatus(master, c, false) != http.StatusNotFound {
return false
}
}
return true
}, 30*time.Second, 200*time.Millisecond, "the replaced chunks were never deleted")
require.Eventually(t, func() bool {
resp, err := mergeVacuumHTTP.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=0.0001", master))
if err != nil {
return false
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false
}
for _, c := range replaced {
if mergeChunkStatus(master, c, true) != http.StatusNotFound {
return false
}
}
return true
}, 4*time.Minute, time.Second, "vacuum left the replaced chunks on disk")
if unflushedWrite {
_, err = held.WriteAt(mergePagePattern(mergePages-1, 99), int64((mergePages-1)*mergePageSize))
require.NoError(t, err)
version[mergePages-1] = 99
}
// A second descriptor shares the handle the first one keeps open. Drop its
// pages from the kernel cache so every read goes to the mount.
reader, err := os.Open(path)
require.NoError(t, err)
defer reader.Close()
require.NoError(t, unix.Fadvise(int(reader.Fd()), 0, 0, unix.FADV_DONTNEED))
started := time.Now()
buf := make([]byte, mergePageSize)
for p := 0; p < mergePages; p++ {
n, err := reader.ReadAt(buf, int64(p*mergePageSize))
require.NoError(t, err, "read page %d through the open handle after the merge", p)
require.Equal(t, mergePageSize, n, "page %d", p)
require.Equal(t, mergePagePattern(p, version[p]), buf, "page %d", p)
}
assert.Less(t, time.Since(started), 5*time.Second, "reads through the open handle waited on chunks the merge replaced")
}
func mergePagePattern(page, version int) []byte {
b := make([]byte, mergePageSize)
for i := 0; i < len(b); i += 8 {
binary.LittleEndian.PutUint32(b[i:], uint32(page))
binary.LittleEndian.PutUint32(b[i+4:], uint32(version))
}
return b
}
type mergeChunk struct {
FileId string `json:"file_id"`
ModifiedTsNs int64 `json:"modified_ts_ns"`
Fid *struct {
VolumeId uint32 `json:"volume_id"`
FileKey uint64 `json:"file_key"`
Cookie uint32 `json:"cookie"`
} `json:"fid"`
}
func (c mergeChunk) fileId() string {
if c.FileId != "" || c.Fid == nil {
return c.FileId
}
return fmt.Sprintf("%d,%x%08x", c.Fid.VolumeId, c.Fid.FileKey, c.Fid.Cookie)
}
// mergeEntryChunks returns the chunks the filer stores for the file.
func mergeEntryChunks(t *testing.T, framework *FuseTestFramework, name string) []mergeChunk {
t.Helper()
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/%s?metadata=true", framework.GetFilerAddr(), url.PathEscape(name)))
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode, "filer metadata of %s", name)
var entry struct {
Chunks []mergeChunk `json:"chunks"`
}
require.NoError(t, json.NewDecoder(resp.Body).Decode(&entry))
return entry.Chunks
}
// mergeChunkStatus HEADs a chunk on the volume server holding it, and returns
// 0 when it cannot ask. With readDeleted, a deleted needle still answers until
// a vacuum removes it.
func mergeChunkStatus(master string, c mergeChunk, readDeleted bool) int {
fid := c.fileId()
volumeId := strings.SplitN(fid, ",", 2)[0]
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/dir/lookup?volumeId=%s", master, volumeId))
if err != nil {
return 0
}
var lookup struct {
Locations []struct {
Url string `json:"url"`
} `json:"locations"`
}
err = json.NewDecoder(resp.Body).Decode(&lookup)
resp.Body.Close()
if err != nil || len(lookup.Locations) == 0 {
return 0
}
target := fmt.Sprintf("http://%s/%s", lookup.Locations[0].Url, fid)
if readDeleted {
target += "?readDeleted=true"
}
head, err := mergeHTTP.Head(target)
if err != nil {
return 0
}
head.Body.Close()
return head.StatusCode
}
-58
View File
@@ -57,61 +57,3 @@ class VectorInstallerTest(unittest.TestCase):
ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual((dest / name).read_bytes(), contents)
class EnterpriseInstallerTest(unittest.TestCase):
shell = VectorInstallerTest.shell
def enterprise(self, code, **env):
return self.shell('ENTERPRISE=true; REPO=seaweedfs/artifactory; EDITION=-enterprise; ' + code, **env)
def test_asset_names(self):
result = self.enterprise('go_asset_name linux amd64; go_asset_name windows amd64; '
'rust_asset_name linux arm64; worker_asset_name linux amd64; '
'vector_asset_name linux arm64; LARGE_DISK=true; '
'go_asset_name darwin arm64; rust_asset_name linux amd64')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.split(), [
'weed-enterprise-linux_amd64.tar.gz', 'weed-enterprise-windows_amd64.zip',
'weed-volume-enterprise_linux_arm64.tar.gz', 'weed-worker-enterprise_linux_amd64.tar.gz',
'weed-vector-enterprise_linux_arm64.tar.gz', 'weed-enterprise-darwin_arm64_large_disk.tar.gz',
'weed-volume-enterprise_large_disk_linux_amd64.tar.gz',
])
def test_open_source_go_asset_names(self):
result = self.shell('go_asset_name linux amd64; go_asset_name windows amd64; '
'LARGE_DISK=true; go_asset_name windows amd64')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.split(), [
'linux_amd64.tar.gz', 'windows_amd64.zip', 'windows_amd64_large_disk.zip',
])
def test_windows_weed_keeps_exe(self):
with tempfile.TemporaryDirectory() as directory:
directory = Path(directory)
archive = directory / 'weed.zip'
with zipfile.ZipFile(archive, 'w') as output:
output.writestr('weed.exe', b'weed fixture')
dest = directory / 'bin'
result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; '
'INSTALL_DIR="$DEST"; install_component weed windows amd64',
ARCHIVE=archive.as_posix(), DEST=dest.as_posix())
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual((dest / 'weed.exe').read_bytes(), b'weed fixture')
def test_rust_binaries_linux_only(self):
for system, arch, supported in [
('linux', 'amd64', True), ('linux', 'arm64', True),
('darwin', 'arm64', False), ('windows', 'amd64', False),
]:
with self.subTest(system=system, arch=arch):
for check in ['vector_platform_supported', 'rust_volume_platform_supported']:
result = self.enterprise(f'{check} "$SYSTEM" "$ARCH"', SYSTEM=system, ARCH=arch)
self.assertEqual(result.returncode == 0, supported, check)
def test_latest_version_skips_other_releases(self):
tags = ['vfs-0.2.5', 'kvcache-filer-native-v0.1.0-preview.1', '4.54-rc1', '4.53', '4.52']
listing = ''.join(f' "tag_name": "{tag}",\\n' for tag in tags)
result = self.enterprise(f"curl() {{ printf '{listing}'; }}; get_latest_version")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.strip(), '4.53')
@@ -1,116 +0,0 @@
//go:build !windows
package metadata_subscribe
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// A peer removal is broadcast only to the filers connected to the master that
// saw the peer go. A peer that dies while no master is up (or across a leader
// change) is never removed: the filer keeps retrying its address, the peer's
// delivery watermark stays at zero, and every aggregated subscriber of that
// filer is held at the settled horizon (2 x LogFlushInterval) for good.
func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
weedBinary := findWeedBinary()
require.NotEmpty(t, weedBinary, "weed binary not found")
testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_")
require.NoError(t, err)
t.Cleanup(func() {
if t.Failed() {
t.Logf("logs kept at %s", testDir)
return
}
os.RemoveAll(testDir)
})
ports, err := testutil.AllocateMiniPorts(3)
require.NoError(t, err)
masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
masterArgs := []string{"master",
"-ip=127.0.0.1",
"-port=" + strconv.Itoa(masterPort),
"-mdir=" + mkdir(t, testDir, "master"),
"-peers=none"}
masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
// filer1 follows filer2: an entry created on filer2 reaches filer1
createPeerEntry(t, filer2Address, "from-filer2")
require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second),
"filer1 never followed filer2")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
events := make(chan *filer_pb.SubscribeMetadataResponse, 1000)
sinceNs := time.Now().UnixNano()
go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events)
arrived := make(map[string]time.Time)
waitArrival := func(name string, within time.Duration) bool {
deadline := time.After(within)
for {
if _, ok := arrived[name]; ok {
return true
}
select {
case ev := <-events:
if n := ev.GetEventNotification().GetNewEntry(); n != nil {
arrived[n.Name] = time.Now()
}
case <-deadline:
return false
}
}
}
createPeerEntry(t, filer1Address, "baseline")
require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber")
// filer2 dies while no master is up, so no removal is ever broadcast
stopProcess(masterProcess)
stopProcess(filer2)
startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second))
diedAt := time.Now()
// Unpatched, every event from here on arrives 2 x LogFlushInterval late.
// The dead peer may hold reads for its unreachable bound plus the removal
// grace (2 x LogFlushInterval each); after that delivery must be prompt.
for i := 0; ; i++ {
name := fmt.Sprintf("probe-%d", i)
createPeerEntry(t, filer1Address, name)
if waitArrival(name, 10*time.Second) {
t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second))
return
}
if time.Since(diedAt) > 6*time.Minute {
t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second))
}
}
}
+1 -3
View File
@@ -104,9 +104,7 @@ for i in $(seq 1 60); do
if mountpoint -q "${MOUNT_DIR}"; then
break
fi
# Not kill -0: sudo briefly runs with a root real uid while starting, so
# signalling it fails with EPERM even though it is alive.
if ! ps -p "${mount_pid}" >/dev/null; then
if ! kill -0 "${mount_pid}" 2>/dev/null; then
echo "weed mount exited early; log tail:" >&2
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
exit 1
+2 -15
View File
@@ -2,16 +2,13 @@ package acl
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -66,22 +63,12 @@ func TestBucketOwnershipControlsLifecycle(t *testing.T) {
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
deleted, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
raw, ok := awsmiddleware.GetRawResponse(deleted.ResultMetadata).(*smithyhttp.Response)
require.True(t, ok)
assert.Equal(t, http.StatusNoContent, raw.StatusCode)
_, err = client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
Bucket: aws.String(bucketName),
})
require.Error(t, err)
assert.Contains(t, err.Error(), "OwnershipControlsNotFoundError")
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerPreferred)
assert.Equal(t, types.ObjectOwnershipBucketOwnerPreferred, getObjectOwnership(t, client, bucketName))
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
}
// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
-1
View File
@@ -1 +0,0 @@
.work
-87
View File
@@ -1,87 +0,0 @@
# Runs the listing tools and listing strategies of
# https://github.com/varveio/s3-listing-study against a local SeaweedFS.
#
# make test build weed, fetch the study and its pinned tools, start
# weed server on free ports, run, stop
# make run ENDPOINT=... run against an already running endpoint
#
# Needs: go, python3, node/npm, cargo, docker (swath image), curl.
#
# The server runs with -s3.allowBackslashInKey, the AWS-compatible key mode: by
# default SeaweedFS rewrites '\' in object keys to '/', and the edge key set
# holds a key with a backslash.
STUDY_REPO ?= https://github.com/varveio/s3-listing-study.git
STUDY_REV ?= 666ec620d245e16baaaa0cf53461382f42c35917
WORK_DIR ?= $(CURDIR)/.work
STUDY_DIR ?= $(WORK_DIR)/s3-listing-study
TOOLS_DIR ?= $(WORK_DIR)/tools
VENV ?= $(WORK_DIR)/venv
WEED_BINARY ?= $(WORK_DIR)/weed
TOOLS ?= aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath
ACCESS_KEY ?= listing_study_access
SECRET_KEY ?= listing_study_secret
JOBS ?= 4
ONLY ?=
.PHONY: test run build-weed study venv tools start-server stop-server clean
test: build-weed tools
@$(MAKE) --no-print-directory start-server; status=$$?; \
if [ $$status -eq 0 ]; then \
$(MAKE) --no-print-directory run ENDPOINT=http://127.0.0.1:$$(cat $(WORK_DIR)/s3.port); status=$$?; \
fi; \
$(MAKE) --no-print-directory stop-server; exit $$status
build-weed:
mkdir -p $(WORK_DIR)
cd ../../../weed && go build -buildvcs=false -o $(WEED_BINARY) .
study:
@if [ "$$(git -C $(STUDY_DIR) rev-parse HEAD 2>/dev/null)" != "$(STUDY_REV)" ]; then \
git init -q $(STUDY_DIR) && \
git -C $(STUDY_DIR) fetch -q --depth 1 $(STUDY_REPO) $(STUDY_REV) && \
git -C $(STUDY_DIR) checkout -q --force FETCH_HEAD; \
fi
venv:
@test -x $(VENV)/bin/python || python3 -m venv $(VENV)
$(VENV)/bin/pip install -q -r requirements.txt
tools: study venv
$(VENV)/bin/python install_tools.py --study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --tools $(TOOLS)
start-server:
@mkdir -p $(WORK_DIR)
@set -e; data=$$(mktemp -d $(WORK_DIR)/data.XXXXXX); ports=$$(python3 free_ports.py 5); \
set -- $$ports; \
$(WEED_BINARY) server -filer -s3 -ip 127.0.0.1 -ip.bind 127.0.0.1 -dir=$$data \
-master.peers=none -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=50 -volume.preStopSeconds=1 \
-master.port=$$1 -volume.port=$$2 -filer.port=$$3 -s3.port=$$4 -metricsPort=$$5 \
-s3.port.iceberg=0 -s3.port.lance=0 -s3.allowBackslashInKey -s3.config=$(CURDIR)/s3.json \
> $(WORK_DIR)/weed.log 2>&1 & echo $$! > $(WORK_DIR)/weed.pid; \
echo $$4 > $(WORK_DIR)/s3.port; \
for i in $$(seq 1 90); do \
if curl -sf -o /dev/null http://127.0.0.1:$$3/ && curl -s -o /dev/null http://127.0.0.1:$$4/; then \
echo "SeaweedFS S3 ready on port $$4"; exit 0; fi; \
kill -0 $$(cat $(WORK_DIR)/weed.pid) || { tail -50 $(WORK_DIR)/weed.log; exit 1; }; \
sleep 1; \
done; tail -50 $(WORK_DIR)/weed.log; exit 1
stop-server:
@if [ -f $(WORK_DIR)/weed.pid ]; then \
pid=$$(cat $(WORK_DIR)/weed.pid); kill $$pid 2>/dev/null || true; \
for i in $$(seq 1 20); do kill -0 $$pid 2>/dev/null || break; sleep 0.5; done; \
rm -f $(WORK_DIR)/weed.pid; \
fi
run: study
$(VENV)/bin/python run_study.py --endpoint $(ENDPOINT) \
--access-key $(ACCESS_KEY) --secret-key $(SECRET_KEY) \
--study $(STUDY_DIR) --tools-dir $(TOOLS_DIR) --work-dir $(WORK_DIR)/run \
--tools $(TOOLS) --jobs $(JOBS) --expected-failures expected_failures.txt \
$(if $(ONLY),--only '$(ONLY)')
clean: stop-server
rm -rf .work
@@ -1,29 +0,0 @@
# Cases that fail for reasons outside SeaweedFS. A listed case that passes is
# reported as XPASS and fails the run, so this list has to shrink with fixes.
# Format: <case name> # reason
# aws-cli prints "None" for a --query over a page with no Contents (the canary
# has no root-level keys); the capsule normalizer reads it as a malformed row.
aws-cli:s3api-v2-remainder:canary:/ # capsule normalizer cannot parse aws-cli's "None"
# The edge bucket holds zero-byte directory markers (markers/dir/, markers/empty/).
# These modes list them as directories, not objects: rclone's --files-only and
# lsf --files-only drop them, s5cmd types any key ending in "/" as DIR and the
# capsule normalizer drops DIR rows from its recursive modes. The reference
# probes check that SeaweedFS returns both markers as Contents.
rclone:recursive-fastlist:edge:/ # tool hides directory marker objects
rclone:recursive-hierarchical:edge:/ # tool hides directory marker objects
rclone:recursive-walk:edge:/ # tool hides directory marker objects
rclone:listv1:edge:/ # tool hides directory marker objects
rclone:lsf:edge:/ # tool hides directory marker objects
s5cmd:recursive:edge:/ # tool hides directory marker objects
s5cmd:listv1:edge:/ # tool hides directory marker objects
s5cmd:allversions:edge:/ # tool hides directory marker objects
# Deleting the last key under a prefix leaves its filer directory behind, and an
# empty directory is still listed as a CommonPrefix until the filer's empty-folder
# cleanup removes it (2 minutes by default). This is deliberate: directories made
# through mount or the filer stay visible to S3 clients (dirHoldsOnlyHiddenEntries
# in weed/s3api/s3api_object_handlers_list.go). AWS derives CommonPrefixes from
# keys only, so prefix-discovery tools briefly see the deleted prefix.
reference:study-churn:deleted-prefixes # empty directories are listed until cleanup
-27
View File
@@ -1,27 +0,0 @@
#!/usr/bin/env python3
"""Print N free TCP ports whose +10000 gRPC companions are free too, all below
the Linux ephemeral range (32768+) that outgoing connections draw from."""
import random
import socket
import sys
def free(port: int) -> bool:
with socket.socket() as s:
try:
s.bind(("127.0.0.1", port))
return True
except OSError:
return False
picked: list[int] = []
taken: list[int] = []
while len(picked) < int(sys.argv[1]):
port = random.randrange(10000, 22000)
pair = (port, port + 10000)
if all(abs(p - t) > 20 for p in pair for t in taken) and all(free(p) for p in pair):
picked.append(port)
taken += pair
print(*picked)
-247
View File
@@ -1,247 +0,0 @@
#!/usr/bin/env python3
"""Install the listing tools pinned by the s3-listing-study checkout.
Every artifact locator and sha256 comes from the study's own
tools/<tool>/build/image.json, so bumping STUDY_REV bumps the tools with it.
Tools land under TOOLS_DIR at the paths listed in run_study.PATH_MAP.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import stat
import subprocess
import sys
import tarfile
import urllib.request
import zipfile
from pathlib import Path
ALL_TOOLS = (
"aws-cli",
"minio-mc",
"ps3",
"rclone",
"s3-fast-list",
"s3kor",
"s3p",
"s4cmd",
"s5cmd",
"s7cmd",
"swath",
)
MC_COMMIT = "d6541ea280b73a834b64d4097e21f2be77676104"
"""github.com/minio/mc tag RELEASE.2025-08-13T08-35-41Z."""
SOURCE_PATCHES = {
"s3-fast-list": (
"s3-fast-list/src/data_map.rs",
(
(
" match ctx.data_map_channel.try_recv() {",
" let list_tasks_running = ctx.all_list_tasks_is_running();\n"
" match ctx.data_map_channel.try_recv() {",
),
(
"!ctx.all_list_tasks_is_running() && !has_more_in_queue",
"!list_tasks_running && !has_more_in_queue",
),
),
),
}
"""Source fixes applied before a build. s3-fast-list's data map reads the channel
and then the list-task state, so a last page sent and completed between the two
reads is dropped and the run exits 0 with a short or empty listing. Reading the
state first means a finished task's pages are already in the channel."""
def patch_source(tool: str, work: Path) -> None:
rel, edits = SOURCE_PATCHES[tool]
path = work / rel
text = path.read_text()
for old, new in edits:
if text.count(old) != 1:
raise SystemExit(f"{tool}: {rel} no longer matches its source patch: {old!r}")
text = text.replace(old, new)
path.write_text(text)
def fetch(url: str, sha256: str, dest: Path) -> Path:
if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == sha256:
return dest
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(dest.suffix + ".part")
request = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
with urllib.request.urlopen(request, timeout=300) as response, tmp.open("wb") as out:
shutil.copyfileobj(response, out)
digest = hashlib.sha256(tmp.read_bytes()).hexdigest()
if digest != sha256:
tmp.unlink()
raise SystemExit(f"{url}: sha256 {digest} != pinned {sha256}")
tmp.rename(dest)
return dest
def executable(path: Path) -> None:
path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
def wrapper(path: Path, body: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"#!/bin/sh\n{body}\n")
executable(path)
def tar_member(archive: Path, member: str, dest: Path) -> None:
with tarfile.open(archive) as tar:
source = tar.extractfile(member)
if source is None:
raise SystemExit(f"{archive}: no member {member}")
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(source.read())
executable(dest)
def install(tool: str, study: Path, tools_dir: Path) -> None:
image = json.loads((study / "tools" / tool / "build" / "image.json").read_text())
artifact = image["tool_artifact"]
url, sha = artifact["locator"], artifact["sha256"]
dl = tools_dir / "downloads"
bin_dir = tools_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
name = url.rsplit("/", 1)[-1]
stamp = tools_dir / "installed" / tool
installed = sha + ("+patched" if tool in SOURCE_PATCHES else "")
if stamp.exists() and stamp.read_text() == installed:
print(f"{tool} {image.get('tool_version', '')} already installed")
return
if tool == "aws-cli":
archive = fetch(url, sha, dl / name)
work = tools_dir / "aws-cli-src"
shutil.rmtree(work, ignore_errors=True)
with zipfile.ZipFile(archive) as zf:
zf.extractall(work)
for rel in ("aws/install", "aws/dist/aws", "aws/dist/aws_completer"):
executable(work / rel)
subprocess.run(
[
str(work / "aws" / "install"),
"--install-dir",
str(tools_dir / "aws-cli"),
"--bin-dir",
str(tools_dir / "aws-cli-bin"),
"--update",
],
check=True,
)
shutil.rmtree(work)
elif tool == "minio-mc":
# dl.min.io answers 410 Gone for every archived mc release, so build the
# same release tag from source instead.
if image["tool_version"] != "RELEASE.2025-08-13T08-35-41Z":
raise SystemExit(f"minio-mc: no source commit pinned for {image['tool_version']}")
subprocess.run(
["go", "install", f"github.com/minio/mc@{MC_COMMIT}"],
env={**os.environ, "GOBIN": str(bin_dir)},
check=True,
)
elif tool == "ps3":
shutil.copyfile(fetch(url, sha, dl / name), bin_dir / "pS3")
executable(bin_dir / "pS3")
elif tool == "rclone":
archive = fetch(url, sha, dl / name)
with zipfile.ZipFile(archive) as zf:
member = f"{name.removesuffix('.zip')}/rclone"
(bin_dir / "rclone").write_bytes(zf.read(member))
executable(bin_dir / "rclone")
elif tool == "s3kor":
tar_member(fetch(url, sha, dl / name), "s3kor", bin_dir / "s3kor")
elif tool == "s5cmd":
tar_member(fetch(url, sha, dl / name), "s5cmd", bin_dir / "s5cmd")
elif tool == "s7cmd":
tar_member(fetch(url, sha, dl / name), "s7cmd", bin_dir / "s7cmd")
elif tool == "s3p":
work = tools_dir / "s3p"
work.mkdir(parents=True, exist_ok=True)
for f in ("package.json", "package-lock.json"):
shutil.copyfile(study / "tools/s3p/build" / f, work / f)
subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], cwd=work, check=True)
wrapper(bin_dir / "s3p", f'exec node "{work}/node_modules/s3p/bin/s3p" "$@"')
elif tool == "s4cmd":
archive = fetch(url, sha, dl / name)
venv = tools_dir / "s4cmd-venv"
subprocess.run([sys.executable, "-m", "venv", "--clear", str(venv)], check=True)
subprocess.run(
[
str(venv / "bin" / "python"),
"-m",
"pip",
"install",
"--quiet",
"--require-hashes",
"--no-deps",
"-r",
str(study / "tools/s4cmd/build/requirements.txt"),
],
check=True,
)
script = tools_dir / "s4cmd" / "s4cmd.py"
with tarfile.open(archive) as tar:
source = tar.extractfile("s4cmd-2.1.0/s4cmd.py")
assert source is not None
script.parent.mkdir(parents=True, exist_ok=True)
script.write_bytes(source.read())
wrapper(bin_dir / "s4cmd", f'exec "{venv}/bin/python" "{script}" "$@"')
elif tool == "s3-fast-list":
archive = fetch(url, sha, dl / name)
work = tools_dir / "s3-fast-list-src"
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True)
subprocess.run(["tar", "xzf", str(archive), "--strip-components=1", "-C", str(work)], check=True)
shutil.copyfile(study / "tools/s3-fast-list/build/Cargo.lock", work / "Cargo.lock")
patch_source(tool, work)
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=work, check=True)
for exe in ("s3-fast-list", "ks-tool"):
shutil.copyfile(work / "target/release" / exe, bin_dir / exe)
executable(bin_dir / exe)
elif tool == "swath":
ref = f"{url}@sha256:{sha}"
subprocess.run(["docker", "pull", "--quiet", ref], check=True, stdout=subprocess.DEVNULL)
cid = subprocess.run(
["docker", "create", ref], check=True, capture_output=True, text=True
).stdout.strip()
try:
dest = tools_dir / "swath"
shutil.rmtree(dest, ignore_errors=True)
dest.mkdir(parents=True)
subprocess.run(["docker", "cp", f"{cid}:/opt/java/openjdk", str(dest / "openjdk")], check=True)
subprocess.run(["docker", "cp", f"{cid}:/opt/swath/swath.jar", str(dest / "swath.jar")], check=True)
finally:
subprocess.run(["docker", "rm", cid], check=True, stdout=subprocess.DEVNULL)
else:
raise SystemExit(f"unknown tool {tool}")
stamp.parent.mkdir(parents=True, exist_ok=True)
stamp.write_text(installed)
print(f"installed {tool} {image.get('tool_version', '')}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--study", required=True, type=Path)
parser.add_argument("--tools-dir", required=True, type=Path)
parser.add_argument("--tools", default=",".join(ALL_TOOLS))
args = parser.parse_args()
for tool in filter(None, args.tools.split(",")):
install(tool, args.study.resolve(), args.tools_dir.resolve())
return 0
if __name__ == "__main__":
os.umask(0o022)
raise SystemExit(main())
-6
View File
@@ -1,6 +0,0 @@
# Versions the pinned s3-listing-study uv.lock resolves for the capsule normalizers.
duckdb==1.5.5
ijson==3.5.1
pyyaml==6.0.3
zstandard==0.25.0
botocore==1.44.5
-922
View File
@@ -1,922 +0,0 @@
#!/usr/bin/env python3
"""Run the s3-listing-study tool capsules and ListObjects reference probes against SeaweedFS.
Tool cases: every mode of every capsule under the study's tools/ is compiled by
the capsule's own command.py (benchmark.adapters.compile_command, with the
custom-endpoint wiring the study's replay-endpoint pilot established), run
against SeaweedFS with the pinned tool build, normalized by the capsule's own
normalize.py, and the normalized listing is compared exactly against the keys
that were written.
Reference probes: raw, signed ListObjectsV2/ListObjects requests implementing
the strategies the study classifies tools by (serial pagination, delimiter
prefix discovery, start-after range splitting) plus the request semantics in
docs/s3-reference.md, checked against the byte-ordered key set.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import time
import traceback
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from pathlib import Path
REGION = "us-east-1"
S3NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
def canary_keys() -> dict[str, int]:
"""benchmark/fixtures/replay-canary/generate.sql: 2,048 keys in 16 prefixes."""
return {f"group-{i // 128:02d}/object-{i:06d}.dat": 1024 + i for i in range(2048)}
EDGE_KEYS = [
# docs/smoke-bucket.md: a root key beside the prefixes, for delimiter checks.
"index.html",
# docs/s3-reference.md examples.
"test_file(3).png",
"photos/2006/February/sample.jpg",
"photos/2006/January/sample.jpg",
"photos/2007/sample.jpg",
"notes/summer/july",
"notes/summer/august",
"notes/winter",
# Byte order around the '/' (0x2f) delimiter.
"order/a",
"order/a b",
"order/a!b",
"order/a-b",
"order/a.b",
"order/a/b",
"order/a/c/d",
"order/a0",
"order/aa",
"order/a~",
"order/A",
"order/b",
# Zero-byte directory markers.
"markers/dir/",
"markers/dir/child",
"markers/empty/",
# Characters that need XML or URL escaping.
"chars/space key",
"chars/plus+key",
"chars/percent%20key",
"chars/amp&key",
"chars/lt<gt>key",
'chars/quote"key',
"chars/apos'key",
"chars/hash#key",
"chars/question?key",
"chars/equals=key",
"chars/semicolon;key",
"chars/colon:key",
"chars/at@key",
"chars/comma,key",
"chars/star*key",
"chars/bang!key",
"chars/dollar$key",
"chars/caret^key",
"chars/backslash\\key",
"chars/brace{key}",
"chars/pipe|key",
"chars/backtick`key",
"chars/tilde~key",
# UTF-8 byte order, which differs from UTF-16 order above U+FFFF.
"unicode/café-nfc",
"unicode/café-nfd",
"unicode/日本語.txt",
"unicode/。-halfwidth",
"unicode/\U0001f600-emoji",
"unicode/z-ascii",
"deep/a/b/c/d/e/f/g/h/i/j/k.txt",
"long/" + "/".join(["y" * 200] * 4),
]
def edge_keys() -> dict[str, int]:
return {k: (0 if k.endswith("/") else 100 + i) for i, k in enumerate(EDGE_KEYS)}
def body_for(key: str, size: int) -> bytes:
seed = hashlib.sha256(key.encode()).digest()
return (seed * (size // len(seed) + 1))[:size]
def utf8_sorted(keys) -> list[str]:
return sorted(keys, key=lambda k: k.encode("utf-8"))
# ---------------------------------------------------------------- signed HTTP
class S3:
def __init__(self, endpoint: str, access: str, secret: str) -> None:
from botocore.auth import S3SigV4Auth
from botocore.credentials import Credentials
self.endpoint = endpoint.rstrip("/")
self.auth = S3SigV4Auth(Credentials(access, secret), "s3", REGION)
def request(
self, method: str, path: str, params: list[tuple[str, str]] | None = None, body: bytes = b""
) -> tuple[int, bytes]:
from botocore.awsrequest import AWSRequest
query = "&".join(
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
for k, v in (params or [])
)
url = self.endpoint + urllib.parse.quote(path, safe="/-_.~") + (f"?{query}" if query else "")
req = AWSRequest(method=method, url=url, data=body)
req.headers["x-amz-content-sha256"] = hashlib.sha256(body).hexdigest()
self.auth.add_auth(req)
prepared = req.prepare()
http = urllib.request.Request(prepared.url, data=body or None, method=method)
for k, v in prepared.headers.items():
http.add_header(k, v)
try:
with urllib.request.urlopen(http, timeout=60) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as err:
return err.code, err.read()
def put_object(self, bucket: str, key: str, body: bytes) -> None:
# The first writes to a new bucket can get a retryable 500 while its
# volumes grow; SDK clients retry those the same way.
for attempt in range(10):
status, out = self.request("PUT", f"/{bucket}/{key}", body=body)
if status < 500:
break
time.sleep(0.5 * (attempt + 1))
if status != 200:
raise RuntimeError(f"PUT {bucket}/{key!r}: {status} {out[:300]!r}")
def delete_object(self, bucket: str, key: str) -> None:
status, out = self.request("DELETE", f"/{bucket}/{key}")
if status != 204:
raise RuntimeError(f"DELETE {bucket}/{key!r}: {status} {out[:300]!r}")
def create_bucket(self, bucket: str) -> None:
status, out = self.request("PUT", f"/{bucket}")
if status not in (200, 409):
raise RuntimeError(f"create bucket {bucket}: {status} {out[:300]!r}")
def list_page(self, bucket: str, params: list[tuple[str, str]]) -> ET.Element:
status, out = self.request("GET", f"/{bucket}", params)
if status != 200:
raise AssertionError(f"list {params}: HTTP {status} {out[:300]!r}")
return ET.fromstring(out)
def text(el: ET.Element, name: str) -> str | None:
child = el.find(S3NS + name)
return None if child is None else (child.text or "")
@dataclass
class Page:
entries: list[tuple[str, str]] # ("key"|"prefix", value) in response order
truncated: bool
token: str | None
next_marker: str | None
key_count: str | None
raw: ET.Element
def check_byte_order(values: list[str], what: str) -> None:
for a, b in zip(values, values[1:]):
assert a.encode() <= b.encode(), f"{what} out of byte order: {a!r} before {b!r}"
def parse_page(root: ET.Element, url_encoded: bool) -> Page:
dec = urllib.parse.unquote_plus if url_encoded else (lambda s: s)
entries: list[tuple[str, str]] = []
for child in root:
if child.tag == S3NS + "Contents":
entries.append(("key", dec(text(child, "Key") or "")))
elif child.tag == S3NS + "CommonPrefixes":
entries.append(("prefix", dec(text(child, "Prefix") or "")))
nm = text(root, "NextMarker")
# XML carries Contents and CommonPrefixes as two lists, each in byte order;
# their merged byte order is what pagination has to respect.
for kind in ("key", "prefix"):
check_byte_order([v for k, v in entries if k == kind], f"{kind} list")
entries.sort(key=lambda e: e[1].encode())
return Page(
entries,
text(root, "IsTruncated") == "true",
text(root, "NextContinuationToken"),
dec(nm) if nm is not None else None,
text(root, "KeyCount"),
root,
)
# ---------------------------------------------------------------- expectations
def direct_view(keys: list[str], prefix: str, delimiter: str) -> list[tuple[str, str]]:
"""What a delimiter listing of prefix returns, in byte order."""
out: dict[str, str] = {}
for k in keys:
if not k.startswith(prefix):
continue
rest = k[len(prefix) :]
idx = rest.find(delimiter) if delimiter else -1
if idx >= 0:
out[prefix + rest[: idx + len(delimiter)]] = "prefix"
else:
out[k] = "key"
return [(out[k], k) for k in utf8_sorted(out)]
# ---------------------------------------------------------------- reference probes
class Probe:
def __init__(self, s3: S3, bucket: str, keys: list[str]) -> None:
self.s3, self.bucket, self.keys = s3, bucket, utf8_sorted(keys)
def v2_all(self, base: list[tuple[str, str]], max_keys: int, url: bool = False) -> list[Page]:
pages: list[Page] = []
token = None
while True:
params = [("list-type", "2"), ("max-keys", str(max_keys)), *base]
if url:
params.append(("encoding-type", "url"))
if token:
params.append(("continuation-token", token))
page = parse_page(self.s3.list_page(self.bucket, params), url)
pages.append(page)
n = len(page.entries)
assert n <= max_keys, f"page holds {n} > max-keys {max_keys}"
assert page.key_count == str(n), f"KeyCount {page.key_count} != {n} entries"
if not page.truncated:
assert not page.token, "NextContinuationToken on an untruncated page"
return pages
assert page.token, "IsTruncated without NextContinuationToken"
assert len(pages) < 10000, "pagination does not terminate"
token = page.token
def v1_all(self, base: list[tuple[str, str]], max_keys: int, delimiter: str) -> list[Page]:
pages: list[Page] = []
marker = None
while True:
params = [("max-keys", str(max_keys)), *base]
if delimiter:
params.append(("delimiter", delimiter))
if marker is not None:
params.append(("marker", marker))
page = parse_page(self.s3.list_page(self.bucket, params), False)
pages.append(page)
assert len(page.entries) <= max_keys
if not page.truncated:
return pages
if delimiter:
assert page.next_marker, "truncated delimiter page without NextMarker"
marker = page.next_marker
else:
marker = page.next_marker or page.entries[-1][1]
assert len(pages) < 10000, "pagination does not terminate"
@staticmethod
def flat(pages: list[Page]) -> list[tuple[str, str]]:
return [e for p in pages for e in p.entries]
def expect_eq(self, got, want, what: str) -> None:
if got != want:
gs, ws = set(got), set(want)
missing = [w for w in want if w not in gs][:5]
extra = [g for g in got if g not in ws][:5]
first = next((i for i, (a, b) in enumerate(zip(got, want)) if a != b), min(len(got), len(want)))
raise AssertionError(
f"{what}: got {len(got)} want {len(want)}; missing {missing} extra {extra}; "
f"first difference at {first}: got {got[first:first + 3]} want {want[first:first + 3]}"
)
# Serial pagination (aws-cli, mc, s3kor, s5cmd ls).
def serial(self, prefix: str, max_keys: int) -> None:
want = [("key", k) for k in self.keys if k.startswith(prefix)]
got = self.flat(self.v2_all([("prefix", prefix)], max_keys))
self.expect_eq(got, want, f"serial prefix={prefix!r} max-keys={max_keys}")
# Delimiter view and page boundaries inside it.
def delimiter(self, prefix: str, delimiter: str, max_keys: int, v1: bool = False) -> None:
want = direct_view(self.keys, prefix, delimiter)
if v1:
got = self.flat(self.v1_all([("prefix", prefix)], max_keys, delimiter))
else:
got = self.flat(self.v2_all([("prefix", prefix), ("delimiter", delimiter)], max_keys))
self.expect_eq(got, want, f"{'v1' if v1 else 'v2'} delimiter={delimiter!r} prefix={prefix!r} max-keys={max_keys}")
# Prefix discovery (rclone walk, s7cmd, s4cmd): recurse through CommonPrefixes.
def walk(self, max_keys: int, v1: bool = False) -> None:
found: list[str] = []
queue = [""]
while queue:
prefix = queue.pop()
if v1:
entries = self.flat(self.v1_all([("prefix", prefix)], max_keys, "/"))
else:
entries = self.flat(self.v2_all([("prefix", prefix), ("delimiter", "/")], max_keys))
for kind, value in entries:
(queue.append if kind == "prefix" else found.append)(value)
self.expect_eq(utf8_sorted(found), self.keys, f"{'v1' if v1 else 'v2'} delimiter walk max-keys={max_keys}")
assert len(found) == len(set(found)), "walk returned a key twice"
# Range splitting (S3P, s3-fast-list, Swath): start-after cut points.
def ranges(self, cuts: list[str], max_keys: int) -> None:
cuts = sorted(set(cuts), key=lambda c: c.encode())
bounds = [None, *cuts, None]
found: list[str] = []
for lo, hi in zip(bounds, bounds[1:]):
token = None
done = False
while not done:
params = [("list-type", "2"), ("max-keys", str(max_keys))]
if token:
params.append(("continuation-token", token))
elif lo is not None:
params.append(("start-after", lo))
page = parse_page(self.s3.list_page(self.bucket, params), False)
for _, key in page.entries:
if lo is not None:
assert key.encode() > lo.encode(), f"start-after {lo!r} returned {key!r}"
if hi is not None and key.encode() > hi.encode():
done = True
break
found.append(key)
if not page.truncated:
break
token = page.token
self.expect_eq(found, self.keys, f"start-after ranges ({len(cuts)} cuts, max-keys={max_keys})")
def start_after_with_delimiter(self, prefix: str, start_after: str) -> None:
want = [
e
for e in direct_view(self.keys, prefix, "/")
if e[1].encode() > start_after.encode()
]
got = self.flat(
self.v2_all([("prefix", prefix), ("delimiter", "/"), ("start-after", start_after)], 1000)
)
self.expect_eq(got, want, f"start-after={start_after!r} delimiter=/ prefix={prefix!r}")
def token_overrides_start_after(self) -> None:
first = parse_page(self.s3.list_page(self.bucket, [("list-type", "2"), ("max-keys", "5")]), False)
assert first.truncated and first.token
params = [("list-type", "2"), ("max-keys", "5"), ("continuation-token", first.token)]
plain = parse_page(self.s3.list_page(self.bucket, params), False)
mixed = parse_page(
self.s3.list_page(self.bucket, [*params, ("start-after", self.keys[len(self.keys) // 2])]),
False,
)
self.expect_eq(mixed.entries, plain.entries, "continuation-token with a later start-after")
def versions(self, prefix: str, delimiter: str, max_keys: int, url: bool) -> None:
"""ListObjectVersions paging (key-marker/version-id-marker) over unversioned keys."""
want = direct_view(self.keys, prefix, delimiter) if delimiter else [("key", k) for k in self.keys if k.startswith(prefix)]
got: list[tuple[str, str]] = []
key_marker = version_marker = None
dec = urllib.parse.unquote_plus if url else (lambda v: v)
for _ in range(10000):
params = [("versions", ""), ("max-keys", str(max_keys)), ("prefix", prefix)]
if delimiter:
params.append(("delimiter", delimiter))
if url:
params.append(("encoding-type", "url"))
if key_marker is not None:
params += [("key-marker", key_marker), ("version-id-marker", version_marker or "")]
root = self.s3.list_page(self.bucket, params)
page: list[tuple[str, str]] = []
for child in root:
if child.tag in (S3NS + "Version", S3NS + "DeleteMarker"):
page.append(("key", dec(text(child, "Key") or "")))
elif child.tag == S3NS + "CommonPrefixes":
page.append(("prefix", dec(text(child, "Prefix") or "")))
assert len(page) <= max_keys, f"{len(page)} entries > max-keys {max_keys}"
for kind in ("key", "prefix"):
check_byte_order([v for k, v in page if k == kind], f"version {kind} list")
got += sorted(page, key=lambda e: e[1].encode())
if text(root, "IsTruncated") != "true":
break
nk = text(root, "NextKeyMarker")
assert nk, "truncated versions page without NextKeyMarker"
key_marker, version_marker = dec(nk), text(root, "NextVersionIdMarker")
self.expect_eq(got, want, f"versions prefix={prefix!r} delimiter={delimiter!r} max-keys={max_keys} url={url}")
def url_encoding(self, prefix: str) -> None:
want = [("key", k) for k in self.keys if k.startswith(prefix)]
got = self.flat(self.v2_all([("prefix", prefix)], 7, url=True))
self.expect_eq(got, want, f"encoding-type=url prefix={prefix!r}")
root = self.s3.list_page(
self.bucket,
[("list-type", "2"), ("encoding-type", "url"), ("prefix", prefix), ("delimiter", "/"), ("start-after", prefix + "a b")],
)
for name, value in (("Prefix", prefix), ("Delimiter", "/"), ("StartAfter", prefix + "a b")):
raw = text(root, name)
assert raw is not None and urllib.parse.unquote_plus(raw) == value, f"encoded {name} {raw!r} != {value!r}"
assert text(root, "EncodingType") == "url", "EncodingType not echoed"
def deleted_prefixes(s3: S3, bucket: str) -> None:
"""A prefix whose last key was deleted is no longer a CommonPrefix."""
s3.create_bucket(bucket)
keys = ["keep", "gone/a/b", "gone/c", "half/x", "half/y/z"]
for k in keys:
s3.put_object(bucket, k, b"x")
for k in ("gone/a/b", "gone/c", "half/y/z"):
s3.delete_object(bucket, k)
p = Probe(s3, bucket, ["keep", "half/x"])
p.delimiter("", "/", 1000)
p.delimiter("half/", "/", 1000)
p.delimiter("gone/", "/", 1000)
def run_probes(
s3: S3, buckets: dict[str, dict[str, int]], churn_bucket: str, only: re.Pattern | None
) -> list[tuple[str, str | None]]:
results: list[tuple[str, str | None]] = []
def check(name: str, fn, *a) -> None:
if only and not only.search(name):
return
try:
fn(*a)
results.append((name, None))
except Exception as exc: # noqa: BLE001
results.append((name, f"{type(exc).__name__}: {exc}"))
check(f"reference:{churn_bucket}:deleted-prefixes", deleted_prefixes, s3, churn_bucket)
for bucket, keymap in buckets.items():
p = Probe(s3, bucket, list(keymap))
keys = p.keys
tag = f"reference:{bucket}"
for mk in (1, 7, 100, 1000):
check(f"{tag}:serial:max-keys={mk}", p.serial, "", mk)
check(f"{tag}:url-encoding", p.url_encoding, "")
check(f"{tag}:token-overrides-start-after", p.token_overrides_start_after)
for mk in (2, 1000):
check(f"{tag}:walk-v2:max-keys={mk}", p.walk, mk)
check(f"{tag}:walk-v1:max-keys={mk}", p.walk, mk, True)
prefixes = sorted({k[: k.index("/") + 1] for k in keys if "/" in k})
scopes = ["", *prefixes[:: max(1, len(prefixes) // 4)]]
if bucket.endswith("edge"):
scopes += ["order/", "order/a", "chars/", "unicode/", "markers/", "markers/dir/", "photos/2006/", "o"]
for prefix in scopes:
for mk in (1, 3, 1000):
check(f"{tag}:delimiter-v2:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk)
check(f"{tag}:delimiter-v1:prefix={prefix}:max-keys={mk}", p.delimiter, prefix, "/", mk, True)
for prefix in ("", scopes[-1]):
for delim in ("", "/"):
for mk in (2, 1000):
check(f"{tag}:versions:prefix={prefix}:delimiter={delim}:max-keys={mk}", p.versions, prefix, delim, mk, False)
check(f"{tag}:versions-url:prefix={prefix}", p.versions, prefix, "/", 3, True)
for d in ("-", "a", "ou"):
check(f"{tag}:delimiter={d}", p.delimiter, "", d, 1000)
check(f"{tag}:serial:prefix-no-slash", p.serial, keys[len(keys) // 3][:4], 5)
# Cut points that are existing keys, absent keys between keys, prefixes,
# and a common prefix with and without its trailing delimiter.
step = max(1, len(keys) // 9)
existing = keys[step::step]
absent = [k + "\x7f" for k in existing[::2]] + [k[:-1] for k in existing[1::2] if len(k) > 1]
dirs = [k[: k.rfind("/") + 1] for k in existing if "/" in k]
dirs += [d.rstrip("/") for d in dirs]
for name, cuts in (("existing", existing), ("absent", absent), ("dirs", dirs), ("mixed", existing + absent + dirs)):
for mk in (3, 1000):
check(f"{tag}:ranges-{name}:max-keys={mk}", p.ranges, cuts, mk)
for sa in (keys[len(keys) // 2], prefixes[len(prefixes) // 2], prefixes[len(prefixes) // 2].rstrip("/")):
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "", sa)
if bucket.endswith("edge"):
for sa in ("order/a", "order/a/", "order/a/b", "order/a.b", "order/a0"):
check(f"{tag}:start-after-delimiter:{sa}", p.start_after_with_delimiter, "order/", sa)
check(f"{tag}:url-encoding:chars/", p.url_encoding, "chars/")
return results
# ---------------------------------------------------------------- tool capsules
@dataclass
class Case:
tool: str
mode: str
bucket: str
prefix: str
status: str = ""
detail: str = ""
argv: list[str] = field(default_factory=list)
class ToolRunner:
def __init__(self, args, study: Path, work: Path, endpoint: str) -> None:
sys.path.insert(0, str(study / "benchmark" / "src"))
from benchmark import adapters # noqa: PLC0415
from benchmark.runtime.command_adapter import CommandAdapterError, Stated # noqa: PLC0415
self.adapters, self.refusal, self.stated = adapters, CommandAdapterError, Stated
self.study, self.work, self.endpoint = study, work, endpoint
tools = Path(args.tools_dir).resolve()
self.fixture_dir = work / "fixtures"
self.path_map = {
"/usr/local/bin/aws": str(tools / "aws-cli-bin" / "aws"),
"/usr/bin/mc": str(tools / "bin" / "mc"),
"/usr/local/bin/pS3": str(tools / "bin" / "pS3"),
"/usr/local/bin/rclone": str(tools / "bin" / "rclone"),
"/usr/bin/s3-fast-list": str(tools / "bin" / "s3-fast-list"),
"/usr/bin/ks-tool": str(tools / "bin" / "ks-tool"),
"/usr/local/bin/s3kor": str(tools / "bin" / "s3kor"),
"/usr/local/bin/s3p": str(tools / "bin" / "s3p"),
"/usr/local/bin/s4cmd": str(tools / "bin" / "s4cmd"),
"/s5cmd": str(tools / "bin" / "s5cmd"),
"/usr/local/bin/s7cmd": str(tools / "bin" / "s7cmd"),
"/opt/java/openjdk/bin/java": str(tools / "swath" / "openjdk" / "bin" / "java"),
"/opt/swath/swath.jar": str(tools / "swath" / "swath.jar"),
"/usr/bin/python3": sys.executable,
"/opt/benchmark/tools/": str(study / "tools") + "/",
}
os.environ["PYTHONPATH"] = str(study / "benchmark" / "src")
self.env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"AWS_ACCESS_KEY_ID": args.access_key,
"AWS_SECRET_ACCESS_KEY": args.secret_key,
"AWS_REGION": REGION,
"AWS_DEFAULT_REGION": REGION,
"AWS_EC2_METADATA_DISABLED": "true",
"TZ": "UTC",
"LC_ALL": "C.UTF-8",
"PYTHONPATH": str(study / "benchmark" / "src"),
}
def localize(self, argv, bucket: str) -> list[str]:
out = []
path_map = {**self.path_map, "/fixtures/source/": f"{self.fixture_dir / bucket}/"}
for token in argv:
for src, dst in path_map.items():
if token == src or (src.endswith("/") and token.startswith(src)):
token = dst + token[len(src) :]
break
out.append(token)
return out
def adapter_dir(self, tool: str) -> Path:
return self.study / "tools" / tool / "adapter"
def write_fixture_companions(self, bucket: str, keys: list[str]) -> None:
"""The staged-fixture companions, built the way fixture_bundle.py builds them."""
import duckdb # noqa: PLC0415
out = self.fixture_dir / bucket
shutil.rmtree(out, ignore_errors=True)
out.mkdir(parents=True)
shards = sorted({k.split("/", 1)[0] for k in keys})
if all(re.fullmatch(r"[A-Za-z0-9._/-]+", s) for s in shards):
(out / "s5cmd-shards.input").write_text("".join(f"{s}\n" for s in shards))
parts = out / "parquet"
parts.mkdir()
with duckdb.connect() as con:
con.execute("CREATE TABLE f (key BLOB, row_type VARCHAR)")
con.executemany("INSERT INTO f VALUES (?, 'OBJECT')", [(k.encode(),) for k in keys])
con.execute(f"COPY f TO '{parts / 'part-00000.parquet'}' (FORMAT parquet)")
sys.path.insert(0, str(self.study / "tools" / "s3-fast-list" / "adapter"))
from fixture_hints import FixtureHintsError, generate_hints # noqa: PLC0415
try:
generate_hints(parts, out / "s3-fast-list-hints.input", segments=4)
except FixtureHintsError as exc:
print(f"{bucket}: no s3-fast-list fixture hints: {exc}", flush=True)
def inject_endpoint(self, tool: str, argv: list[str], env: dict) -> tuple[list[str], dict]:
"""Add the endpoint the way the replay-endpoint pilot (run.sh) did, for modes
the capsule refuses only because the replay server lacks the API."""
ep = self.endpoint
if tool == "rclone":
return [a.replace(f"region={REGION}", f'region={REGION},endpoint="{ep}",force_path_style=true') for a in argv], env
if tool == "minio-mc":
return argv, {**env, "MC_HOST_s3": ep, "MC_REGION": REGION}
if tool == "s3kor":
return [argv[0], argv[1], "--custom-endpoint-url", ep, *argv[2:]], env
if tool == "s7cmd":
return [*argv[:-1], "--target-endpoint-url", ep, "--target-force-path-style", argv[-1]], env
if tool == "s5cmd":
return [argv[0], "--endpoint-url", ep, *argv[1:]], env
return [*argv, "--endpoint-url", ep], env
def config_for(self, adapter, case: Case, mode: str, keys: list[str]) -> dict[str, object]:
config: dict[str, object] = {}
for name, axis in adapter.modes[mode].axes.items():
if isinstance(axis, self.stated):
config[name] = 4
if adapter.tool == "s5cmd" and mode == "fanout-with-dirs":
initials = sorted({k[len(case.prefix)] for k in keys if k.startswith(case.prefix) and len(k) > len(case.prefix)})
if not all(re.fullmatch(r"[A-Za-z0-9._-]", c) for c in initials):
raise NotApplicable(f"key initials {initials} are outside the capsule's safe shard set")
config["shard_initials"] = "".join(initials)
return config
def execute(self, adapter, case: Case, mode: str, sink: Path, keys: list[str], artifact_path: str = "") -> tuple[Path | None, dict]:
signed = adapter.supports_signed
inject = case.tool in INJECT_ENDPOINT and mode in INJECT_ENDPOINT[case.tool]
config = self.config_for(adapter, case, mode, keys)
argv, env = self.adapters.compile_command(
self.adapter_dir(case.tool),
case.tool,
mode=mode,
bucket=case.bucket,
region=REGION,
prefix=case.prefix,
signed=signed,
config=config,
sink_dir=str(sink),
artifact_path=artifact_path,
visible_memory_gb=2.0,
endpoint_url="" if inject else self.endpoint,
)
if inject:
argv, env = self.inject_endpoint(case.tool, list(argv), env)
case.detail = "endpoint injected; "
argv = self.localize(argv, case.bucket)
case.argv = argv
sink.mkdir(parents=True, exist_ok=True)
home = sink / "home"
home.mkdir(exist_ok=True)
manifest = adapter.modes[mode]
stdout_path = sink / (manifest.product_file if manifest.product_channel == "stdout" and manifest.product_artifact else "stdout")
with stdout_path.open("wb") as out, (sink / "stderr").open("wb") as err:
proc = subprocess.run(
argv,
stdout=out,
stderr=err,
env={**self.env, **env, "HOME": str(home)},
cwd=sink,
timeout=300,
check=False,
)
if proc.returncode != 0:
tail = (sink / "stderr").read_bytes()[-600:].decode(errors="replace")
raise RuntimeError(f"exit {proc.returncode}: {tail}")
product = sink / manifest.product_file if manifest.product_artifact else None
return product, config
def run_case(self, case: Case, expected: dict[str, tuple[int, str]]) -> None:
adapter = self.adapters.load_adapter(self.adapter_dir(case.tool), case.tool)
manifest = adapter.modes[case.mode]
if manifest.purpose_ceiling == "preparation" or not manifest.product_artifact or (case.tool, case.mode) in NOT_A_LISTING:
case.status = "skip"
case.detail = f"{manifest.purpose_ceiling} mode, not a compared listing"
return
if (case.tool, case.mode) in ROOT_ONLY and case.prefix:
case.status, case.detail = "skip", "mode lists the bucket root whatever the prefix"
return
fixture = self.fixture_dir / case.bucket
needs = {"fanout-fixture-with-dirs": "s5cmd-shards.input", "list-hinted-fixture": "s3-fast-list-hints.input"}.get(case.mode)
if needs and not (fixture / needs).exists():
case.status, case.detail = "skip", f"fixture_bundle.py would not build {needs} for this key set"
return
keys = list(expected)
sink = self.work / "cases" / case.tool / case.mode / (case.bucket + "~" + (case.prefix.replace("/", "_") or "root"))
shutil.rmtree(sink, ignore_errors=True)
try:
artifact_path = ""
for step in adapter.requires.get(case.mode, ()):
producer_sink = sink / f"requires-{step.mode}"
self.execute(adapter, case, step.mode, producer_sink, keys)
artifact_path = str(producer_sink / adapter.modes[step.mode].artifacts[step.artifact])
if manifest.inline:
inline_sink = sink / f"inline-{manifest.inline}"
self.execute(adapter, case, manifest.inline, inline_sink, keys, artifact_path)
inline_manifest = adapter.modes[manifest.inline]
artifact_path = str(inline_sink / next(iter(inline_manifest.artifacts.values())))
product, config = self.execute(adapter, case, case.mode, sink, keys, artifact_path)
except NotApplicable as exc:
case.status, case.detail = "skip", str(exc)
return
except self.adapters.AdapterError as exc:
if "custom endpoint" in str(exc) or "replay endpoint" in str(exc) or "no --prefix" in str(exc) or "requires HeadBucket" in str(exc):
case.status, case.detail = "skip", f"capsule refuses: {str(exc).split(': ', 2)[-1][:160]}"
return
raise
normalized = sink / "normalized.tsv"
self.adapters.normalize_to_path(
self.adapter_dir(case.tool),
case.tool,
case.mode,
case.prefix,
normalized,
input_path=None if manifest.product_channel == "dataset" else product,
dataset=product if manifest.product_channel == "dataset" else None,
config=config,
)
compare(case, normalized.read_bytes(), expected, manifest.fields)
INJECT_ENDPOINT = {
"aws-cli": {"s3api-v1-text", "s3api-versions-text"},
"minio-mc": {"versions-json", "find", "find-json"},
"rclone": {"listv1"},
"s3kor": {"list-versions"},
"s4cmd": {"recursive", "shallow", "show-directory", "du"},
"s5cmd": {"listv1", "allversions"},
"s7cmd": {"all-versions"},
}
"""Capsule modes refused only because the study's replay server serves
ListObjectsV2 alone; SeaweedFS serves ListObjects, ListObjectVersions and
HeadBucket too."""
NOT_A_LISTING = {
("rclone", "debug"),
("rclone", "walk-debug"),
("s3p", "summarize"),
("s4cmd", "du"),
("s4cmd", "show-directory"),
}
"""Modes with no per-key listing: rclone's normalizer has no debug modes, summarize
and du print one aggregate, and `ls -d` prints the directory itself."""
class NotApplicable(Exception):
pass
ROOT_ONLY = {
("aws-cli", "s3api-v2-remainder"),
("s5cmd", "fanout-fixture-with-dirs"),
("s3-fast-list", "list-hinted-fixture"),
}
"""Modes whose capsule lists the root (remainder) or a staged root-scope fixture."""
SHALLOW = re.compile(r"shallow|delimiter|rootkeys|remainder|max-depth")
KEYS_ONLY = re.compile(r"rootkeys|remainder")
"""Shallow modes whose normalizer keeps only the keys, dropping CommonPrefixes."""
def compare(case: Case, normalized: bytes, expected: dict[str, tuple[int, str]], fields) -> None:
rows = []
for line in normalized.split(b"\n"):
if not line:
continue
f = line.split(b"\t")
rows.append((f[0].decode("utf-8", "surrogateescape"), f[1].decode(), f[2].decode()))
scope = {k: v for k, v in expected.items() if k.startswith(case.prefix)}
keys = utf8_sorted(scope)
shallow = bool(SHALLOW.search(case.mode))
if shallow:
view = direct_view(keys, case.prefix, "/")
want_objects = [k for kind, k in view if kind == "key"]
allowed_dirs = {k for kind, k in view if kind == "prefix"}
else:
want_objects = keys
allowed_dirs = {k[: i + 1] for k in keys for i, c in enumerate(k) if c == "/" and i >= len(case.prefix)}
problems: list[str] = []
objects, dirs = [], []
for key, size, etag in rows:
is_dir = key.endswith("/") and size == "-" and etag == "-" and key not in scope
(dirs if is_dir else objects).append((key, size, etag))
got_keys = [k for k, _, _ in objects]
dup = {k for k in got_keys if got_keys.count(k) > 1} if len(got_keys) != len(set(got_keys)) else set()
if dup:
problems.append(f"duplicate keys {sorted(dup)[:5]}")
missing = [k for k in want_objects if k not in set(got_keys)]
extra = [k for k in set(got_keys) if k not in set(want_objects)]
if missing:
problems.append(f"missing {len(missing)}: {missing[:5]}")
if extra:
problems.append(f"unexpected {len(extra)}: {sorted(extra)[:5]}")
bad_dirs = [d for d, _, _ in dirs if d not in allowed_dirs]
if bad_dirs:
problems.append(f"unexpected directory rows {bad_dirs[:5]}")
if shallow and not KEYS_ONLY.search(case.mode) and {d for d, _, _ in dirs} != allowed_dirs:
problems.append(f"common prefixes {sorted(d for d, _, _ in dirs)[:8]} != {sorted(allowed_dirs)[:8]}")
for key, size, etag in objects:
if key not in scope:
continue
want_size, want_etag = scope[key]
if size != "-" and size != str(want_size):
problems.append(f"{key!r} size {size} != {want_size}")
if etag != "-" and etag != want_etag:
problems.append(f"{key!r} etag {etag} != {want_etag}")
case.status = "fail" if problems else "pass"
case.detail += "; ".join(problems)[:1500] if problems else f"{len(objects)} objects, {len(dirs)} prefixes"
# ---------------------------------------------------------------- main
def load_expected_failures(path: Path | None) -> dict[str, str]:
out: dict[str, str] = {}
if path and path.exists():
for line in path.read_text().splitlines():
line = line.strip()
if line and not line.startswith("#"):
name, _, reason = line.partition("#")
out[name.strip()] = reason.strip()
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--endpoint", required=True)
ap.add_argument("--access-key", required=True)
ap.add_argument("--secret-key", required=True)
ap.add_argument("--study", required=True)
ap.add_argument("--tools-dir", required=True)
ap.add_argument("--work-dir", required=True)
ap.add_argument("--tools", default="aws-cli,minio-mc,ps3,rclone,s3-fast-list,s3kor,s3p,s4cmd,s5cmd,s7cmd,swath")
ap.add_argument("--bucket-prefix", default="study")
ap.add_argument("--expected-failures", type=Path)
ap.add_argument("--skip-upload", action="store_true")
ap.add_argument("--only", default="", help="regex over case names")
ap.add_argument("--jobs", type=int, default=4)
args = ap.parse_args()
study = Path(args.study).resolve()
work = Path(args.work_dir).resolve()
work.mkdir(parents=True, exist_ok=True)
s3 = S3(args.endpoint, args.access_key, args.secret_key)
buckets = {f"{args.bucket_prefix}-canary": canary_keys(), f"{args.bucket_prefix}-edge": edge_keys()}
expected: dict[str, dict[str, tuple[int, str]]] = {}
for bucket, keymap in buckets.items():
expected[bucket] = {k: (size, hashlib.md5(body_for(k, size)).hexdigest()) for k, size in keymap.items()}
if args.skip_upload:
continue
s3.create_bucket(bucket)
start = time.time()
with concurrent.futures.ThreadPoolExecutor(16) as pool:
list(pool.map(lambda k: s3.put_object(bucket, k, body_for(k, keymap[k])), keymap))
print(f"uploaded {len(keymap)} objects to {bucket} in {time.time() - start:.1f}s", flush=True)
only = re.compile(args.only) if args.only else None
results: list[tuple[str, str, str]] = []
for name, err in run_probes(s3, buckets, f"{args.bucket_prefix}-churn", only):
results.append((name, "fail" if err else "pass", err or ""))
runner = ToolRunner(args, study, work, args.endpoint)
cases: list[Case] = []
for bucket in buckets:
runner.write_fixture_companions(bucket, list(buckets[bucket]))
scopes = ["", "group-07/"] if bucket.endswith("canary") else ["", "order/", "chars/"]
for tool in filter(None, args.tools.split(",")):
adapter = runner.adapters.load_adapter(runner.adapter_dir(tool), tool)
for mode in adapter.modes:
for prefix in scopes:
case = Case(tool, mode, bucket, prefix)
if only and not only.search(case_name(case)):
continue
cases.append(case)
def run(case: Case) -> Case:
start = time.time()
try:
runner.run_case(case, expected[case.bucket])
except Exception as exc: # noqa: BLE001
case.status = "fail"
case.detail = f"{type(exc).__name__}: {exc}"[:1500]
if os.environ.get("STUDY_DEBUG"):
traceback.print_exc()
print(f"{case.status:5} {case_name(case)} ({time.time() - start:.1f}s) {case.detail[:300]}", flush=True)
return case
with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
for case in pool.map(run, cases):
results.append((case_name(case), case.status, case.detail))
xfail = load_expected_failures(args.expected_failures)
regressions, fixed = [], []
counts: dict[str, int] = {}
for name, status, detail in results:
if status == "fail" and name in xfail:
status = "xfail"
elif status == "pass" and name in xfail:
fixed.append(name)
elif status == "fail":
regressions.append((name, detail))
counts[status] = counts.get(status, 0) + 1
(work / "results.json").write_text(json.dumps(results, indent=1, ensure_ascii=False))
print("\n== summary:", ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
for name in fixed:
print(f"XPASS (remove from expected failures): {name}")
for name, detail in regressions:
print(f"FAIL {name}: {detail}")
return 1 if regressions or fixed else 0
def case_name(case: Case) -> str:
return f"{case.tool}:{case.mode}:{case.bucket.rsplit('-', 1)[-1]}:{case.prefix or '/'}"
if __name__ == "__main__":
raise SystemExit(main())
Loaded 100 of 485 files, more files were not shown because too many files have changed in this diff. Show more