Compare commits

..
Author SHA1 Message Date
Chris LuandClaude Opus 5.5 b89d7e36bf install.sh: offer the vector engine on linux only
Releases build weed-vector for linux amd64/arm64 only, so --component vector
on darwin or windows downloaded a missing asset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
2026-11-14 11:26:49 -08:00
616 changed files with 5185 additions and 53141 deletions

No files matched your search

+21 -92
View File
@@ -22,16 +22,6 @@ CHART = ROOT / "k8s/charts/seaweedfs"
SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
# No workload is needed to exercise Helm's real lookup and update.
RELEASE_SETTINGS = (
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
"global.seaweedfs.createClusterRole=false",
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
)
CONFIGMAP_CHART = ("seaweedfs", "--repo", "https://seaweedfs.github.io/seaweedfs/helm", "--version", "4.54.0")
def run(*args):
@@ -87,7 +77,8 @@ def fixtures():
def check_generated(value):
assert len(value) == 64 and value.isascii() and value.isalnum(), "invalid generated JWT key"
decoded = base64.b64decode(value, validate=True).decode("ascii")
assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
def check_helpers(helm, reference_helm):
@@ -156,11 +147,20 @@ def check_upgrades(helm, context):
legacy = "seaweedfs-security-config"
kubectl = ["kubectl", "--context", context, "-n", namespace]
release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
for setting in RELEASE_SETTINGS:
# No workload is needed to exercise Helm's real ConfigMap lookup and update.
for setting in (
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
"global.seaweedfs.createClusterRole=false",
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
):
release_args += ["--set", setting]
def stored():
return secret_keys(kubectl, current)
cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
return keys(cm["data"]["security.toml"])
def upgrade():
run(helm, "upgrade", *release_args)
@@ -168,8 +168,8 @@ def check_upgrades(helm, context):
def patch(raw):
# Seed previous-release content without taking Helm 4's SSA ownership.
run(*kubectl, "patch", "secret", current, "--type=merge", "--field-manager=helm", "-p",
json.dumps({"stringData": {"security.toml": raw}}))
run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
json.dumps({"data": {"security.toml": raw}}))
run(*kubectl, "create", "namespace", namespace)
try:
@@ -194,24 +194,17 @@ def check_upgrades(helm, context):
assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
print(f"PASS upgrade: {name}")
# Migration from the ConfigMaps of earlier chart versions, then precedence of the Secret.
configmap_raw = "\n".join(f'[{section}]\nkey = "configmap-{index}"'
for index, section in enumerate(SECTIONS))
run(*kubectl, "create", "configmap", current, "--from-literal=security.toml=" + configmap_raw)
run(*kubectl, "delete", "secret", current)
assert upgrade() == keys(configmap_raw), "ConfigMap keys were not preserved"
print("PASS upgrade: ConfigMap migration")
# Migration from the old chart name, followed by precedence of the current name.
legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
for index, section in enumerate(SECTIONS))
run(*kubectl, "delete", "configmap", current)
run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
run(*kubectl, "delete", "secret", current)
run(*kubectl, "delete", "configmap", current)
assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
print("PASS upgrade: legacy ConfigMap migration")
current_raw = next(fixtures())[1]
patch(current_raw)
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode the Secret"
print("PASS upgrade: Secret takes precedence")
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
print("PASS upgrade: current ConfigMap takes precedence")
patch(ESCAPED_HEADER)
try:
@@ -220,76 +213,13 @@ def check_upgrades(helm, context):
assert "unsupported quoted section header" in error.stderr, error.stderr
else:
raise AssertionError("unsupported quoted header silently rotated its key")
assert stored()["jwt.signing"] == "existing"
assert keys(run(*kubectl, "get", "configmap", current, "-o",
"jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
print("PASS upgrade: unsupported quoted header leaves stored key untouched")
finally:
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
def secret_keys(kubectl, name):
secret = json.loads(run(*kubectl, "get", "secret", name, "-o", "json"))
return keys(base64.b64decode(secret["data"]["security.toml"]).decode())
def check_configmap_release_upgrade(helm, context):
namespace = "jwt-key-migrate-" + uuid.uuid4().hex[:8]
name = "jk-seaweedfs-security-config"
kubectl = ["kubectl", "--context", context, "-n", namespace]
common = ["--kube-context", context, "-n", namespace]
for setting in RELEASE_SETTINGS:
common += ["--set", setting]
upgrade = ["upgrade", "jk", str(CHART), *common, "--set", "master.enabled=true"]
def refuse_partitioned(stage):
try:
run(helm, *upgrade, "--set", "master.updatePartition=1")
except subprocess.CalledProcessError as error:
assert "updatePartition=0" in error.stderr, error.stderr
else:
raise AssertionError(f"partitioned upgrade removed the ConfigMap ({stage})")
def exists(kind):
return run(*kubectl, "get", kind, name, "--ignore-not-found", "-o", "name").strip()
run(*kubectl, "create", "namespace", namespace)
try:
run(helm, "install", "jk", *CONFIGMAP_CHART, *common)
before = keys(run(*kubectl, "get", "configmap", name, "-o", "jsonpath={.data.security\\.toml}"))
assert set(before) == set(SECTIONS), "ConfigMap chart omitted a JWT section"
refuse_partitioned("ConfigMap release")
print("PASS upgrade: partitioned upgrade from the ConfigMap chart is refused")
# Fail an upgrade after Helm created the Secret, so it never removes the ConfigMap.
run(*kubectl, "create", "quota", "no-statefulsets", "--hard=count/statefulsets.apps=0")
try:
run(helm, *upgrade)
except subprocess.CalledProcessError as error:
assert "quota" in error.stderr, error.stderr
else:
raise AssertionError("upgrade was not blocked by the quota")
run(*kubectl, "delete", "quota", "no-statefulsets")
assert exists("secret") and exists("configmap"), "failed upgrade did not leave both objects"
refuse_partitioned("retry after a failed upgrade")
print("PASS upgrade: partitioned retry after a failed migration is refused")
run(helm, "upgrade", "jk", str(CHART), *common)
assert secret_keys(kubectl, name) == before, "upgrade from the ConfigMap chart changed a key"
assert not exists("configmap"), "upgrade left the ConfigMap behind"
print("PASS upgrade: ConfigMap chart release moves its keys to the Secret")
# The unprefixed legacy name can belong to another release in the namespace.
other = "seaweedfs-security-config"
run(*kubectl, "create", "configmap", other, '--from-literal=security.toml=[jwt.signing]\nkey = "other"')
run(*kubectl, "annotate", "configmap", other, "meta.helm.sh/release-name=other",
"meta.helm.sh/release-namespace=" + namespace)
run(helm, *upgrade, "--set", "master.updatePartition=1")
assert secret_keys(kubectl, name) == before, "another release's ConfigMap changed a key"
assert run(*kubectl, "get", "configmap", other, "-o", "name").strip(), "removed another release's ConfigMap"
print("PASS upgrade: another release's legacy ConfigMap does not block a partitioned upgrade")
finally:
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--helm", default="helm")
@@ -300,7 +230,6 @@ def main():
check_helpers(args.helm, args.reference_helm)
if args.context:
check_upgrades(args.helm, args.context)
check_configmap_release_upgrade(args.helm, args.context)
if __name__ == "__main__":
+21 -98
View File
@@ -37,7 +37,7 @@ jobs:
with:
version: v3.18.4
- uses: actions/setup-go@v7
- uses: actions/setup-go@v6
with:
go-version-file: go.mod
@@ -104,51 +104,8 @@ jobs:
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0
echo "S3 rollout settings render correctly"
echo "=== Testing S3 preStopSeconds ==="
if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
--set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then
echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1
fi
grep -q "must be at least 35" /tmp/s3-prestop-short.yaml
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
--set s3.enabled=true --set s3.preStopSeconds=20 \
--set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml
grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
--set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
--set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml
grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml
grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml
grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml
grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml
grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml
echo "S3 preStopSeconds renders correctly"
echo "=== Testing the S3 config checksum is stable across renders ==="
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
# Secret drew new keys for the checksum whenever lookup found nothing (on
@@ -259,40 +216,6 @@ jobs:
grep -q "security-config" /tmp/security.yaml
echo "Security configuration renders correctly"
echo "=== Testing volume server HTTPS ==="
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/security.yaml | grep -q 'enabled = false'
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set global.seaweedfs.volumeHttps.enabled=true \
--set volumes.hot.replicas=1 \
> /tmp/volume-https.yaml
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml > /tmp/volume-https-client.txt
grep -q 'enabled = true' /tmp/volume-https-client.txt
grep -q 'cert = "/usr/local/share/ca-certificates/client/tls.crt"' /tmp/volume-https-client.txt
grep -q 'ca = "/usr/local/share/ca-certificates/ca/tls.crt"' /tmp/volume-https-client.txt
awk '/^ *\[https.volume\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml \
| grep -q 'cert = "/usr/local/share/ca-certificates/volume/tls.crt"'
grep -q "'\*.test-seaweedfs-volume.default'" /tmp/volume-https.yaml
grep -q "'\*.test-seaweedfs-volume-hot.default'" /tmp/volume-https.yaml
test "$(grep -c 'scheme: HTTPS' /tmp/volume-https.yaml)" -eq 4
test "$(grep -c 'checksum/volume-https:' /tmp/volume-https.yaml)" -eq 4
if grep -q 'checksum/volume-https' /tmp/security.yaml; then
echo "FAIL: the volume HTTPS rollout annotation should only render while the option is on"
exit 1
fi
if helm template test $CHART_DIR --set global.seaweedfs.volumeHttps.enabled=true > /dev/null 2>/tmp/volume-https-nosec.err; then
echo "FAIL: volumeHttps without enableSecurity should fail to render"
exit 1
fi
grep -q "requires global.seaweedfs.enableSecurity" /tmp/volume-https-nosec.err
if helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true \
--set global.seaweedfs.volumeHttps.enabled=true --set allInOne.enabled=true > /dev/null 2>/tmp/volume-https-aio.err; then
echo "FAIL: volumeHttps with allInOne should fail to render"
exit 1
fi
grep -q "not supported with allInOne" /tmp/volume-https-aio.err
echo "Volume server HTTPS renders [https.volume], [https.client], certificate names, HTTPS probes and rollout annotations"
echo "=== Testing secure bucket-creation hook certificate mounts ==="
helm template test $CHART_DIR \
--show-only templates/shared/post-install-bucket-hook.yaml \
@@ -394,9 +317,9 @@ jobs:
def docs(manifest):
return [d for d in yaml.safe_load_all(manifest) if d]
def security_config(manifest, name):
def configmap(manifest, name):
for d in docs(manifest):
if d.get("kind") == "Secret" and d["metadata"]["name"] == name:
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
return d
return None
@@ -421,13 +344,13 @@ jobs:
# security-related; preserve that so this PR is non-breaking on
# existing installs.
out = render({})
if security_config(out, "test-seaweedfs-security-config") is not None:
failed.append("defaults: security Secret should not render")
if configmap(out, "test-seaweedfs-security-config") is not None:
failed.append("defaults: security ConfigMap should not render")
else:
print("defaults: no security-config Secret (unchanged)")
print("defaults: no security-config ConfigMap (unchanged)")
# Case 2: filerWrite=true alone is the documented opt-in for
# the Admin UI Users tab. The Secret must render with
# the Admin UI Users tab. Configmap must render with
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
# only exist with mTLS).
out = render({
@@ -437,11 +360,11 @@ jobs:
# refuses to bind without authentication.
"admin.secret.adminPassword": "ci-admin-password",
})
cm = security_config(out, "test-seaweedfs-security-config")
cm = configmap(out, "test-seaweedfs-security-config")
if cm is None:
failed.append("filerWrite=true: security Secret missing")
failed.append("filerWrite=true: security ConfigMap missing")
else:
toml = cm["stringData"]["security.toml"]
toml = cm["data"]["security.toml"]
if "[jwt.filer_signing]" not in toml:
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
if "[grpc" in toml:
@@ -449,7 +372,7 @@ jobs:
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
# Case 3: filer + admin pods must MOUNT the security Secret
# Case 3: filer + admin pods must MOUNT the security ConfigMap
# under filerWrite=true so the JWT key reaches both processes.
# Cert volumes must NOT be present (no mTLS).
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
@@ -469,11 +392,11 @@ jobs:
# with both [jwt.signing] and [grpc.*]. Guards against the
# decoupling change accidentally regressing the mTLS path.
out = render({"global.seaweedfs.enableSecurity": "true"})
cm = security_config(out, "test-seaweedfs-security-config")
cm = configmap(out, "test-seaweedfs-security-config")
if cm is None:
failed.append("enableSecurity=true: security Secret missing")
failed.append("enableSecurity=true: security ConfigMap missing")
else:
toml = cm["stringData"]["security.toml"]
toml = cm["data"]["security.toml"]
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
if missing:
failed.append(f"enableSecurity=true: security.toml missing {missing}")
@@ -490,8 +413,8 @@ jobs:
except subprocess.CalledProcessError as e:
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
continue
if security_config(out, "test-seaweedfs-security-config") is not None:
failed.append(f"{null_path}=null: should not render the security Secret")
if configmap(out, "test-seaweedfs-security-config") is not None:
failed.append(f"{null_path}=null: should not render configmap")
else:
print(f"{null_path}=null: render tolerates explicit null")
@@ -595,16 +518,16 @@ jobs:
mount = next((m for m in pod["containers"][0].get("volumeMounts", []) if m["name"] == "security-config"), None)
# The file has to land on one of weed's config search paths, so
# the target matters as much as the volume itself - and a volume
# of that name backed by anything but the chart's Secret
# of that name backed by anything but the chart's ConfigMap
# leaves `weed shell` without the signing key just the same.
target = ("/etc/seaweedfs/security.toml", "security.toml")
secret = "test-seaweedfs-security-config"
configmap = "test-seaweedfs-security-config"
if vol is None or mount is None:
failed.append("filerWrite=true: bucket hook does not mount security-config (s3.configure would fail)")
elif (mount.get("mountPath"), mount.get("subPath")) != target:
failed.append(f"filerWrite=true: bucket hook mounts security-config as {mount}")
elif vol.get("secret", {}).get("secretName") != secret:
failed.append(f"filerWrite=true: security-config is not backed by the {secret} Secret: {vol}")
elif vol.get("configMap", {}).get("name") != configmap:
failed.append(f"filerWrite=true: security-config is not backed by the {configmap} ConfigMap: {vol}")
else:
print("filerWrite=true: bucket hook mounts security-config at /etc/seaweedfs/security.toml")
@@ -2110,7 +2033,7 @@ jobs:
"--set", "admin.enabled=true", "--set", "admin.secret.adminPassword=ci",
"--set", "cosi.enabled=true", "--set", "networkPolicy.enabled=true",
"--set", "volumes.dc2.replicas=1",
# render the security Secret without its generated jwt key
# render the security ConfigMap without its generated jwt key
"--set", "global.seaweedfs.enableSecurity=true",
"--set", "global.seaweedfs.securityConfig.jwtSigning.volumeWrite=false",
]
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
- uses: actions/setup-python@v7
- uses: actions/setup-python@v6
with:
python-version: '3.13'
- uses: actions/cache/restore@v6
@@ -168,6 +168,9 @@ jobs:
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
- name: Run Rust unit tests (redb experimental cursor)
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
rust-unit-tests-windows:
name: Rust Unit Tests (Windows)
runs-on: windows-latest
@@ -205,6 +208,9 @@ jobs:
- name: Run Rust unit tests
run: cd seaweed-volume && cargo test
- name: Run Rust unit tests (redb experimental cursor)
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
rust-integration-tests:
name: Rust Integration Tests
runs-on: ubuntu-24.04
@@ -1,74 +0,0 @@
name: "S3 listing study tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/listing_study/**'
- '.github/workflows/s3-listing-study-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/listing_study/**'
- '.github/workflows/s3-listing-study-tests.yml'
concurrency:
group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
listing-study:
name: s3-listing-study tools and strategies
runs-on: ubuntu-24.04
timeout-minutes: 40
defaults:
run:
working-directory: test/s3/listing_study
steps:
- name: Check out code
uses: actions/checkout@v7
with:
persist-credentials: false
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Cache pinned listing tools
uses: actions/cache@v6
with:
path: test/s3/listing_study/.work/tools
key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }}
- name: Build weed and install the study's pinned tools
run: make build-weed tools
- name: Run the listing study against SeaweedFS
timeout-minutes: 15
run: make test
- name: Show logs on failure
if: failure()
run: |
tail -200 .work/weed.log || true
- name: Upload results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-listing-study-results
path: |
test/s3/listing_study/.work/run/results.json
test/s3/listing_study/.work/weed.log
retention-days: 14
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ['3.10', '3.11', '3.12']
python-version: ['3.9', '3.11', '3.12']
steps:
- name: Checkout code
+1 -11
View File
@@ -68,15 +68,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
- name: Cache suite Maven dependencies
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: ~/.m2/repository
key: snowflake-s3compat-m2-${{ hashFiles('test/s3/snowflake/run.sh') }}
restore-keys: |
snowflake-s3compat-m2-
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
- name: Install SeaweedFS
run: |
@@ -86,8 +78,6 @@ jobs:
- name: Run Snowflake S3Compat API tests
timeout-minutes: 20
env:
MAVEN_ARGS: -ntp -s ${{ github.workspace }}/test/s3/snowflake/maven-settings.xml
run: |
# Starts weed server, creates the buckets/objects the suite needs,
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
-6
View File
@@ -684,12 +684,6 @@ jobs:
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
pull apache/spark:3.5.1
- name: Cache Iceberg jars
uses: actions/cache@v6
with:
path: ~/.cache/seaweedfs-test-jars
key: iceberg-jars-${{ hashFiles('test/s3tables/catalog_spark/setup_test.go') }}
- name: Run go mod tidy
run: go mod tidy
@@ -1,57 +0,0 @@
name: "S3 Tigris Compatibility Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/tigris_compat/**'
- '.github/workflows/s3-tigris-compat-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/tigris_compat/**'
- '.github/workflows/s3-tigris-compat-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
jobs:
s3-tigris-compat-tests:
name: s3-api-compat-tests (Tigris)
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run s3-api-compat-tests
working-directory: test/s3/tigris_compat
run: make test
- name: Show server log on failure
if: failure()
working-directory: test/s3/tigris_compat
run: tail -200 weed-test.log || true
- name: Upload report
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-tigris-compat-results
path: test/s3/tigris_compat/results.json
if-no-files-found: ignore
retention-days: 7
+4 -4
View File
@@ -50,7 +50,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -410,7 +410,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -581,7 +581,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -810,7 +810,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.10'
python-version: '3.9'
- name: Clone s3-tests
run: |
@@ -6,16 +6,12 @@ on:
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/telemetry-integration.yml'
pull_request:
branches: [ master ]
paths:
- 'telemetry/**'
- 'weed/telemetry/**'
- 'go.mod'
- 'go.sum'
- '.github/workflows/telemetry-integration.yml'
concurrency:
@@ -56,12 +56,8 @@ jobs:
- name: Setup Caddy
run: |
CADDY_VERSION=2.11.9
base="https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}"
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz"
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_checksums.txt"
grep " caddy_${CADDY_VERSION}_linux_amd64.tar.gz$" "caddy_${CADDY_VERSION}_checksums.txt" | sha512sum -c -
tar -xzf "caddy_${CADDY_VERSION}_linux_amd64.tar.gz" caddy
curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
chmod +x caddy
./caddy version
echo "{
auto_https disable_redirects
-2
View File
@@ -67,8 +67,6 @@ Download the latest binary from the [releases](https://github.com/seaweedfs/seaw
curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
```
For SeaweedFS Enterprise binaries, pass `--enterprise` to the script: `... | bash -s -- --enterprise`.
Then start a ready-to-use S3 object store:
```bash
+52 -51
View File
@@ -5,7 +5,7 @@ go 1.26.6
require (
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/pubsub v1.51.1
cloud.google.com/go/storage v1.69.0
cloud.google.com/go/storage v1.67.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.55.8
github.com/beorn7/perks v1.0.1 // indirect
@@ -84,24 +84,24 @@ require (
github.com/xdg-go/scram v1.2.0
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/api/v3 v3.7.3
go.etcd.io/etcd/client/v3 v3.7.3
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
go.mongodb.org/mongo-driver v1.17.9
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.49.0
gocloud.dev/pubsub/natspubsub v0.49.0
gocloud.dev v0.47.0
gocloud.dev/pubsub/natspubsub v0.46.0
gocloud.dev/pubsub/rabbitpubsub v0.47.0
golang.org/x/crypto v0.58.0
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
golang.org/x/image v0.46.0
golang.org/x/net v0.62.0
golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.49.0
golang.org/x/text v0.43.0 // indirect
golang.org/x/tools v0.52.0 // indirect
golang.org/x/tools v0.51.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.310.0
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b // indirect
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 // indirect
google.golang.org/grpc v1.86.0-dev.0.20261027044109-d619b5d02089
google.golang.org/protobuf v1.36.12
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -123,10 +123,10 @@ require (
github.com/apache/iceberg-go v0.7.0
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/aws/aws-sdk-go-v2 v1.48.0
github.com/aws/aws-sdk-go-v2/config v1.33.9
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
github.com/aws/aws-sdk-go-v2 v1.47.7
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/cognusion/imaging v1.0.4
github.com/fluent/fluent-logger-golang v1.10.1
@@ -145,7 +145,7 @@ require (
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.32.0
github.com/pkg/sftp v1.13.12
github.com/rabbitmq/amqp091-go v1.16.0
github.com/rabbitmq/amqp091-go v1.15.0
github.com/rclone/rclone v1.75.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.23.0
@@ -162,7 +162,6 @@ require (
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
go.etcd.io/etcd/client/pkg/v3 v3.7.3
go.uber.org/atomic v1.12.0
go.yaml.in/yaml/v3 v3.0.6
golang.org/x/sync v0.24.0
golang.org/x/tools/godoc v0.1.0-deprecated
google.golang.org/grpc/security/advancedtls v1.0.0
@@ -185,7 +184,7 @@ require (
github.com/anchore/go-lzo v0.1.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/thrift v0.24.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -248,7 +247,7 @@ require (
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/oklog/ulid/v2 v2.1.2 // indirect
github.com/oklog/ulid/v2 v2.1.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
@@ -277,35 +276,37 @@ require (
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.45.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.1 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.6 // indirect
golang.org/x/mod v0.42.0 // indirect
gonum.org/v1/gonum v0.17.0 // indirect
)
require (
cel.dev/expr v0.25.3 // indirect
cloud.google.com/go/auth v0.24.1 // indirect
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a // indirect
cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect
cloud.google.com/go/compute/metadata v0.10.0 // indirect
cloud.google.com/go/iam v1.14.0 // indirect
cloud.google.com/go/monitoring v1.31.0 // indirect
cloud.google.com/go/monitoring v1.30.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.24.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
@@ -319,22 +320,22 @@ require (
github.com/andybalholm/cascadia v1.3.4 // indirect
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6
github.com/aws/smithy-go v1.28.9
github.com/aws/smithy-go v1.28.8
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/buengese/sgzip v0.1.1 // indirect
@@ -369,14 +370,14 @@ require (
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/errors v0.22.9 // indirect
github.com/go-openapi/strfmt v0.27.2 // indirect
github.com/go-openapi/errors v0.22.8 // indirect
github.com/go-openapi/strfmt v0.27.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.3 // indirect
github.com/go-resty/resty/v2 v2.17.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/goccy/go-json v0.11.2 // indirect
github.com/goccy/go-json v0.10.6 // indirect
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
@@ -386,7 +387,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.4.0
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -418,8 +419,8 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nats.go v1.49.0 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -433,7 +434,7 @@ require (
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.33
github.com/pierrec/lz4/v4 v4.1.29
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
@@ -455,7 +456,7 @@ require (
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.9.0 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
github.com/tarantool/go-iproto v1.1.0 // indirect
@@ -480,9 +481,9 @@ require (
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/bbolt v1.5.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
go.opentelemetry.io/otel v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
@@ -492,7 +493,7 @@ require (
go.uber.org/zap v1.28.0 // indirect
golang.org/x/term v0.47.0
golang.org/x/time v0.16.0
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
+112 -110
View File
@@ -4,8 +4,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
cloud.google.com/go/auth v0.24.1 h1:NTmd2iJb4DsGIav0KiqFMYI88nMshA2wnFMBatRmoCs=
cloud.google.com/go/auth v0.24.1/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a h1:jW1ZLFy9oa5ai8LOI40z6s+DL1cO/L1KSm9hVGnibn8=
cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a/go.mod h1:1dmBXa9xfBvBr39ZnDUDkaaA91ccq9CcqENDgJA4d7c=
cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88=
cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk=
cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY=
@@ -18,14 +18,14 @@ cloud.google.com/go/logging v1.20.0 h1:l7yzCB8/JSY5PHT9RsaqdT1jaBGbM6KlSJDX210Hl
cloud.google.com/go/logging v1.20.0/go.mod h1:ALUaJQZM691Eeujy9kFx0xREq3AMPuGd+pTRZuTYv+w=
cloud.google.com/go/longrunning v1.3.0 h1:lZ6c8SJJBmILp/Wnr3vGnBLa0q+cYfSHRCcUfVI2hpI=
cloud.google.com/go/longrunning v1.3.0/go.mod h1:wpgLxCn+UA3/TZ1bj/fW39mfDpwVlMPu1xKJyoMt7V0=
cloud.google.com/go/monitoring v1.31.0 h1:LCuAoYiq8snyL/eDAG/VC73uWFXeax8+YEfmoNGJUrY=
cloud.google.com/go/monitoring v1.31.0/go.mod h1:7uk2dIJIf7ra64dI1VuQ0ttoFqvtKjdzq46u6zNP1hU=
cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY=
cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM=
cloud.google.com/go/pubsub v1.51.1 h1:R3G1wCOxBO7jRpL8x2pdZMv1GAJDF6ax/m2zPOtvTNE=
cloud.google.com/go/pubsub v1.51.1/go.mod h1:y2T0IKtW1iWwVvazYaRpqOAFO4gy2+O7dTDt9TWY/5U=
cloud.google.com/go/pubsub/v2 v2.7.0 h1:MFrBTZZa6PDWZzCi4NJRsHKMm2w0a4oAaYNqwjgbQTE=
cloud.google.com/go/pubsub/v2 v2.7.0/go.mod h1:JaFvWNVRk3Knoil/4M1ECeLOaI9D8drbmJWypQlK5aM=
cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM=
cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g=
cloud.google.com/go/storage v1.67.0 h1:8xD3NvKuHE4c+b/kmSIVBdTc0BiOjplx+KecKGR9fr8=
cloud.google.com/go/storage v1.67.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
@@ -48,8 +48,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3 h1:zoPQ7KXGgzqeUmNWl+b6IwVAqsb5lROEJhqAmqXQtm0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.3/go.mod h1:ei/sDx/dwDIUtZE3PQP1u8+qrA+CRy4q7qW+A2MbgB4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4=
@@ -60,8 +60,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY=
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ=
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA=
github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI=
@@ -75,12 +75,14 @@ github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjw
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
github.com/Files-com/files-sdk-go/v3 v3.3.194/go.mod h1:rl0WumSN9gSo775DgvQv+wMQ8rlb0ES/1hU5jkMtLXg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0 h1:5y2LR2pripbccOte6BpZ+guv5O5W0OQIfKvUYZrGaAQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.62.0/go.mod h1:b3f7EifFRqkVt142USoGIP62uUeZm7ALCPrs1ZjWXYo=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0 h1:tRsXhJnn0w0TNwJpVrB94wsrimzjWKFzdQlUhRdNUzc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.62.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0 h1:aPMa5qZU9dp4fU34o+jQgVIPFIOi20oxYg8TLigmZLg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.62.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0 h1:rmzbFS2CkHq0y2L5LkMVWm+QXjjNEXxjmhrJqga/QGk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.38.0/go.mod h1:iIyrdhveU5Ow84ipRlxtjCOQnIM0DiqSMxuxm50xwZY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0 h1:BeC1Bub7Ttk33TF8v3ZeQ2L8acvz4ZLqtQuLyJAZquc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.61.0/go.mod h1:UgG/Xn9+NXb1/ueSczIFxyuya3hs3srFDiZG2JrpJwY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0 h1:eSgSuKdcpY7RTN1oXSzC6S39rPBImn03ubLMsWK4RRg=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.61.0/go.mod h1:ifhe5teRNcbxg9p353DhOkOgczzL/IBLYFqT1cm1zpM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0 h1:KhpL26/GviYITHMCEGGqT/l50siFaBCBBKrjv6Z0G/k=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.61.0/go.mod h1:sg6Wqbl0BKKZ5zKnQj2q0qic/3ziF894SCx1LpbX37Y=
github.com/IBM/go-sdk-core/v5 v5.23.1 h1:fxLusCG+GlGY1SM7BYAAiiSFhzKUs5zqs0P8knDqeFU=
github.com/IBM/go-sdk-core/v5 v5.23.1/go.mod h1:yO+OQpByKDLTvpEcsFFexgzpeR8eRfCFWAYzxkAu4bk=
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
@@ -171,50 +173,50 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16 h1:q9YuNkYR/7lxflWCcEv3C0cecUt3Zvrh2hn9ZNxieNA=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.16/go.mod h1:3BSeJQF55QQI+dDFHx4WTIjTvX/Lvtk8Jaaf/SGAnBM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5 h1:4EezYQsIY0xPbD6u7TbQLfnpicSMQgA8n9paomVV4Zo=
github.com/aws/aws-sdk-go-v2/service/sns v1.47.5/go.mod h1:Yr4VqXnyoyjWnOWJck8BOidkJtleQ1c8ypgd2n0epxE=
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4 h1:D+Zmdt3T3P770MrqTmZvalbj6ADO7xA7n+qnNakGvrI=
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.4/go.mod h1:dUYH3ia3qEO5ogaWDLp2tjtb4u4Y9xRbae7tOr1ClIA=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1 h1:9T6eO8/WXhij4PQaa2WaYsdekSCmT9Hw66Pw0S8do0Q=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0 h1:fD02zeo2Kc4hlFUvGX3FfA+gGOwd4BBcYgPDHDEzx9Y=
github.com/aws/aws-sdk-go-v2/service/sns v1.46.0/go.mod h1:/lB2ILjoT5/q0zSwtHnrJPjXBvjalnEIl0IgSEdvHec=
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0 h1:FghdY9ebeZf9iIan9J0xfBEID8arYibNBeZ1XHvs4oA=
github.com/aws/aws-sdk-go-v2/service/sqs v1.51.0/go.mod h1:wu7uOJ/2nAdenjvhHikeF6j0al99kfPVG6ZDU4e76OY=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -507,12 +509,12 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-openapi/errors v0.22.9 h1:HI9+SyVYiRzyeBQGv0CbWbQa+u2S0nny7G7AhY+UMiM=
github.com/go-openapi/errors v0.22.9/go.mod h1:R73q66sXP2smzAGgFOI4nqtusRK2xN5pYCjsij+KX58=
github.com/go-openapi/strfmt v0.27.2 h1:SG32SlbwNy92s0KJiVxt2joJeFdqIYHvwrA0OU6HqzQ=
github.com/go-openapi/strfmt v0.27.2/go.mod h1:M4CKsMO0Fb8qR10+1Ra75wCKNNquy+Vj+4LWZrhTo2E=
github.com/go-openapi/testify/v2 v2.8.0 h1:19QDx5b57p8KjO2E4sgs/woL4Akp+FGLRlVm2xn0Rho=
github.com/go-openapi/testify/v2 v2.8.0/go.mod h1:4gUN8jC+RqOE2qjW/vDmfyTvl5BCht4qL7oca0odrak=
github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I=
github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w=
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -544,8 +546,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
@@ -561,8 +563,8 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFig
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
@@ -647,8 +649,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -913,10 +915,10 @@ github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.15 h1:StSf9TINInaZtr4oww2+kXmfwa9SkN//g/LwS19/UJ0=
github.com/nats-io/nats-server/v2 v2.11.15/go.mod h1:zwhv8Y0PE3KHyKgznJc/9Xoai638SaJd83zzJ5GJn74=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -927,8 +929,8 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec=
github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -977,8 +979,8 @@ github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741/go.mod h1:pxMtw7c
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014=
github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c=
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
@@ -1049,8 +1051,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.16.0 h1:1O8oQqnj9hLACoo6CVMgn57z2pLWXvgemhh4GsquV+Y=
github.com/rabbitmq/amqp091-go v1.16.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw=
github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rclone/Proton-API-Bridge v1.0.5 h1:K1++Qtk3PvgkiCCiv6Pahju1TMOzKY6VSwiwT7XLAVc=
github.com/rclone/Proton-API-Bridge v1.0.5/go.mod h1:vCeOPhlXzevN0AFojgh1zsjhetiShy/ArvJ/xkFUDWk=
github.com/rclone/go-proton-api v1.0.4 h1:AJW0e9pB4j0hVK4WqyGErFwaI+5MUQWPCtj5FYYxtPg=
@@ -1153,8 +1155,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4=
github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
github.com/spiffe/go-spiffe/v2 v2.8.2 h1:jUEsvCMD6fH25J8K/w3q/XnIx8W1lb8+YLaEEHIjHmc=
github.com/spiffe/go-spiffe/v2 v2.8.2/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -1349,36 +1351,36 @@ go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
go.etcd.io/etcd/api/v3 v3.7.3 h1:2tumepXe+wi7Wab64uUO71K7ISQyUWLGC1DJbbc59sY=
go.etcd.io/etcd/api/v3 v3.7.3/go.mod h1:HOR9Qn4gb/1/hVAmYprmlLpvTe6ZgfgNS9Z/+Ty65po=
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
go.etcd.io/etcd/client/pkg/v3 v3.7.3 h1:5zMwNu3ycrzCeA5NwOfQbxFRhqq974chq1DrYMua4QI=
go.etcd.io/etcd/client/pkg/v3 v3.7.3/go.mod h1:/OhzqYrF0U7d1NOk55VMyUQGbYD1WoIe4bJwxb47wOk=
go.etcd.io/etcd/client/v3 v3.7.3 h1:UrXcetgkuSEocaBlQlulQRJlXLEnUAHuBRbHh0lue1Q=
go.etcd.io/etcd/client/v3 v3.7.3/go.mod h1:DcPxpqcpV8WIaRi/2waPCx0/Uq8rQUweDJaDYZk1gEE=
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0 h1:Ct6+aaiSO6Nk5SOb6PYcR7iWp+83YF/pL1OJcGqOmbg=
go.opentelemetry.io/contrib/detectors/gcp v1.47.0/go.mod h1:3N259ss0b1AVOs+fYw7lOO0mcUwNqSIpQ+Zitpuqx00=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=
go.opentelemetry.io/contrib/detectors/gcp v1.46.0/go.mod h1:nsrN5c/sOLoY2vsPxN/rQ0V0nvGrWJCqcW4UXLtqNG8=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 h1:MCcYL7J6Vt/X0kjqbMZkekCmwsurbQRbL69vkiye2lk=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0/go.mod h1:3jnStNwSufK+f5ktjL4EPcwtig4rtd81NS70lqHuXl8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0 h1:LaLVGJtIzHcvHjVs2OftmKcc+M9mdOqS8vZ1fBD1Yes=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.47.0/go.mod h1:vF6H5sNDuQBJWgSX00iZgbIqk2MZZoVMH0ar5t5bOpw=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0 h1:qkDYCAFiZXLcs1L4aY+tP2wguQ4kURANqHOQMA2et2s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.46.0/go.mod h1:tkipS4DRzmpAmvg+Gw4++O1IdDq6TVDnvnYU6cmbQVs=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0 h1:RuynHbfU8JUEw7DyONgkVYg2SVtsoF28y0LGIr69jgA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.44.0/go.mod h1:qZF+/lBs71APw8mlnEZcqZHMzqrYrsFiJOv83lX1OGo=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
@@ -1398,8 +1400,8 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syod
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v1.11.1 h1:SCrPqH9NE5CmBrOqpjNCiT0X5mGwMyGERXdogfR7Yjw=
go.opentelemetry.io/proto/otlp v1.11.1/go.mod h1:wq8W4aSf6bOy8RZyESOeQFr9Pu7wydWX/TY4sgvrTPg=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -1425,10 +1427,10 @@ go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
gocloud.dev v0.49.0 h1:02y4IBHX12xiCBK3Nv80IzuCmZXLsOfgpKwMEKzR9O4=
gocloud.dev v0.49.0/go.mod h1:uN+hz2x8ieyW5RjeMetvmpV/nRqFOr7DF0CKVY0Owp8=
gocloud.dev/pubsub/natspubsub v0.49.0 h1:Jk1bywJ6LvqvBbxOQ0qs0q4aa+ORgg1FzjhTy+5O4cI=
gocloud.dev/pubsub/natspubsub v0.49.0/go.mod h1:bmtxptoOPBNAgvTW9tWzoZvw9vzunDoNk6l4cHGd6nc=
gocloud.dev v0.47.0 h1:qN4/0vtutn97f1X5ziisf3WXuaFL9AR0FIRnyJRYzgA=
gocloud.dev v0.47.0/go.mod h1:OdMnyXQRhM17U0a0DwU57pe0c01qN3lJNmaeeRYW0D0=
gocloud.dev/pubsub/natspubsub v0.46.0 h1:cTmJnWOmhb3dYRzX7PvNkq2m5mQdBLFFxZbl/21pfjk=
gocloud.dev/pubsub/natspubsub v0.46.0/go.mod h1:hbEEsNt6mGQoonGQi39VOjHfx2f13DOmWouh4hGsfrU=
gocloud.dev/pubsub/rabbitpubsub v0.47.0 h1:iDSspbLlDbPL2avESbDfMNBK4r3B6H04Ejx+/hqh9JE=
gocloud.dev/pubsub/rabbitpubsub v0.47.0/go.mod h1:2tSZnGnGLPw2ayW4pVaUmnA1LCjP7rekpXPkaBUZt0o=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -1449,8 +1451,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e h1:HgI6W1OolKK6tcV74is+rl9SMT0njsHlp0TbQiGXI/A=
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e/go.mod h1:DBoxwWhjaW8DJJw/YeM4zpeOKMC1FMXSblsYy+1mFSw=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1606,8 +1608,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1629,10 +1631,10 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b h1:+t2qGqt6R8CO622uiX1JpnRvRvSOQoVRV0DiPzUjKXg=
google.golang.org/genproto v0.0.0-20261019180522-ce908d29299b/go.mod h1:XRIfc1yr5xsbF8xFw5+g7SEmLJX/eQ7MMazTRbnMMvI=
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b h1:nB3spRIaSPtxJpxb0k41yljROvm2Jrwfs2pSNdKS5y8=
google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b/go.mod h1:9N4vtjEwvgQKEdlFVQO0M44ErhecQ9DCSQfGoZ51M6w=
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941 h1:CXq2XEi5aEyLjjPZIwqrmQH8ZY4ctUwK7uyAx3mjX/U=
google.golang.org/genproto v0.0.0-20261012220534-4341edf3f941/go.mod h1:3gK0ehnhP0V7/x3ypGoLyT3d0n6TSi/EcsphIjHrcz4=
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 h1:DS/w3N8rsfrwxwgGSj6WhMsPo7aNQGmDx74J81zd0NQ=
google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941/go.mod h1:AuGAdi9KnXss7xbLxIKhCvk+Ro/SxpET/iS45DNYN+M=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k=
google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+31 -80
View File
@@ -7,14 +7,12 @@
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
# curl -fsSL ... | bash -s -- --enterprise --component all
#
# Options:
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
# --version VER Release version tag (default: latest)
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
# --dir DIR Installation directory (default: /usr/local/bin)
# --enterprise Install SeaweedFS Enterprise binaries (from seaweedfs/artifactory)
# --help Show this help message
set -euo pipefail
@@ -26,10 +24,6 @@ LARGE_DISK=false
INSTALL_DIR="/usr/local/bin"
WORKER_INSTALLED=false
VECTOR_INSTALLED=false
VOLUME_INSTALLED=false
ENTERPRISE=false
# Enterprise assets carry this after the binary name, e.g. weed-vector-enterprise_linux_amd64.tar.gz.
EDITION=""
# Colors (if terminal supports them)
if [ -t 1 ]; then
@@ -59,7 +53,6 @@ while [ $# -gt 0 ]; do
--version) VERSION="$2"; shift 2 ;;
--large-disk) LARGE_DISK=true; shift ;;
--dir) INSTALL_DIR="$2"; shift 2 ;;
--enterprise) ENTERPRISE=true; REPO="seaweedfs/artifactory"; EDITION="-enterprise"; shift ;;
--help|-h) usage ;;
*) error "Unknown option: $1. Use --help for usage." ;;
esac
@@ -88,22 +81,15 @@ detect_platform() {
}
# Get latest release tag from GitHub API
# The Enterprise repo also releases other products (vfs-*, previews), which can
# be its latest release, so take the newest plain version tag like 4.53.
get_latest_version() {
local url="https://api.github.com/repos/${REPO}/releases/latest"
if [ "$ENTERPRISE" = true ]; then
url="https://api.github.com/repos/${REPO}/releases?per_page=30"
fi
local body
if command -v curl &>/dev/null; then
body="$(curl -fsSL "$url")"
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
elif command -v wget &>/dev/null; then
body="$(wget -qO- "$url")"
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
else
error "Neither curl nor wget found. Please install one."
fi
echo "$body" | grep '"tag_name"' | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' | grep -E '^[0-9]+(\.[0-9]+)+$' | head -1 || true
}
# Download a file
@@ -124,22 +110,17 @@ go_asset_name() {
if [ "$LARGE_DISK" = true ]; then
suffix="${suffix}_large_disk"
fi
if [ "$ENTERPRISE" = true ]; then
suffix="weed-enterprise-${suffix}"
fi
if [ "$os" = "windows" ]; then
echo "${suffix}.zip"
else
echo "${suffix}.tar.gz"
fi
echo "${suffix}.tar.gz"
}
# Build Rust volume server asset name
rust_asset_name() {
local os="$1" arch="$2"
local prefix="weed-volume${EDITION}"
local prefix="weed-volume"
if [ "$LARGE_DISK" = true ]; then
prefix="${prefix}_large_disk"
prefix="weed-volume_large_disk"
else
prefix="weed-volume"
fi
local suffix="${os}_${arch}"
if [ "$os" = "windows" ]; then
@@ -149,19 +130,6 @@ rust_asset_name() {
fi
}
# Enterprise publishes the Rust volume server for linux only.
rust_volume_platform_supported() {
case "$2" in
amd64|arm64) ;;
*) return 1 ;;
esac
case "$1" in
linux) return 0 ;;
darwin|windows) [ "$ENTERPRISE" != true ] ;;
*) return 1 ;;
esac
}
# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops
# the installer on anything else: a rate limit or a network blip must not read
# as "this release predates the binary" and quietly skip it.
@@ -186,23 +154,19 @@ asset_exists() {
# maintains tables through the namespace and never opens a volume file.
worker_asset_name() {
local os="$1" arch="$2"
echo "weed-worker${EDITION}_${os}_${arch}.tar.gz"
echo "weed-worker_${os}_${arch}.tar.gz"
}
# The vector engine behind the S3 Vectors API, built like the worker.
# The vector engine behind the S3 Vectors API, published for linux only like
# the worker.
vector_asset_name() {
local os="$1" arch="$2"
if [ "$os" = "windows" ]; then
echo "weed-vector${EDITION}_${os}_${arch}.zip"
else
echo "weed-vector${EDITION}_${os}_${arch}.tar.gz"
fi
echo "weed-vector_${os}_${arch}.tar.gz"
}
vector_platform_supported() {
case "$1/$2" in
linux/amd64|linux/arm64) return 0 ;;
darwin/amd64|darwin/arm64|windows/amd64) [ "$ENTERPRISE" != true ] ;;
*) return 1 ;;
esac
}
@@ -222,10 +186,7 @@ install_component() {
download "$download_url" "${tmpdir}/${asset_name}"
info "Extracting ${asset_name}..."
case "$asset_name" in
*.zip) unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" ;;
*) tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" ;;
esac
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
# The Go release action puts the binary inside a directory
local weed_bin
@@ -238,16 +199,20 @@ install_component() {
fi
chmod +x "$weed_bin"
local dest_name="weed"
if [ "$os" = "windows" ]; then
dest_name="weed.exe"
fi
install_binary "$weed_bin" "$dest_name"
ok "Installed weed to ${INSTALL_DIR}/${dest_name}"
install_binary "$weed_bin" "weed"
ok "Installed weed to ${INSTALL_DIR}/weed"
;;
volume-rust)
rust_volume_platform_supported "$os" "$arch" || error "The Rust volume server is not available for ${os}/${arch}"
# Check platform support for Rust volume server
case "$os" in
linux|darwin|windows) ;;
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
esac
case "$arch" in
amd64|arm64) ;;
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
esac
asset_name="$(rust_asset_name "$os" "$arch")"
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
@@ -279,7 +244,6 @@ install_component() {
dest_name="weed-volume.exe"
fi
install_binary "$rust_bin" "$dest_name"
VOLUME_INSTALLED=true
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
;;
@@ -316,26 +280,19 @@ install_component() {
;;
vector)
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}"
vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}. Supported: linux/amd64, linux/arm64"
asset_name="$(vector_asset_name "$os" "$arch")"
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
download "$download_url" "${tmpdir}/${asset_name}"
info "Extracting ${asset_name}..."
local vector_name="weed-vector"
if [ "$os" = "windows" ]; then
command -v unzip >/dev/null || error "unzip is required for Windows vector archives"
unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
vector_name="weed-vector.exe"
else
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
fi
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
local vector_bin
vector_bin="$(find "$tmpdir" -name "$vector_name" -type f | head -1)"
[ -n "$vector_bin" ] || error "Could not find $vector_name binary in archive"
vector_bin="$(find "$tmpdir" -name weed-vector -type f | head -1)"
[ -n "$vector_bin" ] || error "Could not find weed-vector binary in archive"
chmod +x "$vector_bin"
install_binary "$vector_bin" "$vector_name"
install_binary "$vector_bin" weed-vector
VECTOR_INSTALLED=true
ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
;;
@@ -392,11 +349,7 @@ main() {
case "$COMPONENT" in
all)
install_component "weed" "$os" "$arch"
if rust_volume_platform_supported "$os" "$arch"; then
install_component "volume-rust" "$os" "$arch"
else
warn "Skipping the Rust volume server: no build for ${os}/${arch}"
fi
install_component "volume-rust" "$os" "$arch"
# The worker is published for linux amd64/arm64 only, and only by
# releases new enough to carry it; skip either case rather than fail
# an install that has already put two binaries in place.
@@ -425,7 +378,7 @@ main() {
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
info " weed: ${INSTALL_DIR}/weed"
fi
if [ "$VOLUME_INSTALLED" = true ]; then
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
info " weed-volume: ${INSTALL_DIR}/weed-volume"
fi
if [ "$WORKER_INSTALLED" = true ]; then
@@ -438,9 +391,7 @@ main() {
info "Quick start:"
info " weed master # Start master server"
info " weed volume -mserver=localhost:9333 # Start Go volume server"
if [ "$VOLUME_INSTALLED" = true ]; then
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
fi
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
if [ "$WORKER_INSTALLED" = true ]; then
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
fi
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.54"
appVersion: "4.53"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.54.0
version: 4.53.1
+5 -20
View File
@@ -30,12 +30,10 @@ so your deployment will be spread/HA.
The chart's templates render with Helm 3.16.3 and newer versions tested in CI.
Earlier Helm versions are not covered by this chart's compatibility checks.
When JWT signing is enabled, `security.toml` is kept in the
`<fullname>-security-config` Secret. A live upgrade reuses keys from that
Secret, or from the security ConfigMap of earlier chart versions (including
the legacy ConfigMap name); a key absent from them is generated. Plain
`helm template` does not read cluster state; use an install and upgrade
against a cluster to check key persistence.
When JWT signing is enabled, a live upgrade reuses keys from the existing
security ConfigMap (including the legacy ConfigMap name); a key absent from
that ConfigMap is generated. Plain `helm template` does not read cluster state;
use an install and upgrade against a cluster to check key persistence.
The key reader supports single-line quoted TOML strings in JWT sections,
including indented assignments, quoted key names, and bare or simply quoted
@@ -571,8 +569,6 @@ s3:
With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
Outside all-in-one mode, the volume Ingress and the default volume route target the `client-volume` Service, which lists only Ready volume pods. The `volume` Service also publishes pods that are not Ready yet, so their DNS records can exist before readiness. An explicit `backendRefs` entry that names the `volume` Service can therefore reach a volume server before it is Ready. Name the `client-volume` Service instead, with `volume.port`, and copy its `metadata.name` from `helm template` because long release names are truncated.
## Network Policies
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
@@ -614,17 +610,6 @@ Two things worth knowing before you turn this on:
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
## Volume server HTTPS
With `global.seaweedfs.enableSecurity`, gRPC between components uses mutual TLS, but the volume servers' HTTP port, which carries the file data, stays plaintext. Set `global.seaweedfs.volumeHttps.enabled: true` to serve it over TLS:
- `[https.volume]` points at the volume certificate, which then also names every volume group's service (`*.<fullname>-volume.<namespace>`), since volume servers register as `<pod>.<fullname>-volume.<namespace>`. With `certificates.externalCertificates`, the provided `<fullname>-volume-cert` needs those names.
- `[https.client]` is enabled with the client certificate and the chart CA, so every chart pod that talks to the volume servers does it over HTTPS.
- The volume probes use HTTPS.
- Every pod that reads `security.toml` gets a `checksum/volume-https` annotation, so switching the option on or off restarts them together.
The volume servers do not require client certificates, because kubelet probes cannot present one; enable JWT signing (`securityConfig.jwtSigning`) to authorize volume reads and writes. `[https.client]` also sends the admin UI's HTTP lookups on masters and S3 over HTTPS, which their plain listeners refuse, so the UI can miss volume public URLs and refuse SSE-S3 vector buckets. A volume Ingress or HTTPRoute needs its controller's HTTPS-backend setting. `weed mount`, the CSI driver and other clients outside the chart need their own `[https.client]` with the chart CA. The option is not supported with `allInOne`, whose volume server registers by pod IP.
## Common labels
`commonLabels` adds labels to the metadata of every object the chart creates. It may be a map or a template string rendering to one. The chart's own labels cannot be overridden, and pods (see `podLabels`) and `volumeClaimTemplates` are left unchanged.
@@ -733,7 +718,7 @@ The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverri
`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `<fullname>-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets.
The default `vector.metricsPort: 0` keeps TCP probes. Set `vector.metricsPort: 9329` to serve `/metrics`, `/health`, and `/ready` from the engine; readiness then checks master registration. Images older than 4.54 do not accept the metrics flag, so leave metrics off when pinning `image.tag` or `vector.imageOverride` to one. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
To provision vector buckets during installation and upgrades, set:
@@ -56,7 +56,6 @@ spec:
{{- with .Values.admin.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.admin.restartPolicy }}
{{- if .Values.admin.affinity }}
@@ -314,8 +313,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -1,16 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.allInOne.enabled }}
{{- if include "seaweedfs.volumeHttps" . }}
{{- fail "global.seaweedfs.volumeHttps is not supported with allInOne: its volume server registers by pod IP, which the volume certificate cannot name" }}
{{- end }}
{{- $s3PreStopSeconds := 0 }}
{{- if .Values.allInOne.s3.enabled }}
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
{{- if kindIs "invalid" $s3PreStopSeconds }}
{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }}
{{- end }}
{{- $s3PreStopSeconds = $s3PreStopSeconds | int }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -75,7 +64,7 @@ spec:
{{- tpl .Values.allInOne.tolerations . | nindent 8 }}
{{- end }}
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
terminationGracePeriodSeconds: 60
enableServiceLinks: false
{{- if .Values.allInOne.priorityClassName }}
priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
@@ -271,9 +260,6 @@ spec:
{{- if $auditLogConfig }}
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
{{- end }}
{{- if gt $s3PreStopSeconds 0 }}
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
{{- end }}
{{- end }}
{{- if .Values.allInOne.sftp.enabled }}
-sftp \
@@ -507,8 +493,8 @@ spec:
name: {{ include "seaweedfs.fullname" . }}-master-config
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -38,14 +38,6 @@ spec:
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}'
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc'
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc.cluster.local'
{{- if include "seaweedfs.volumeHttps" . }}
{{- range $vname, $_ := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
{{- $volumeService := printf "%s.%s" (include "seaweedfs.componentName" (list $ (trimSuffix "-" (printf "volume-%s" $vname)))) (include "seaweedfs.namespace" $) }}
- '*.{{ $volumeService }}'
- '*.{{ $volumeService }}.svc'
- '*.{{ $volumeService }}.svc.cluster.local'
{{- end }}
{{- end }}
{{- if .Values.certificates.ipAddresses }}
ipAddresses:
{{- range .Values.certificates.ipAddresses }}
@@ -39,7 +39,6 @@ spec:
{{- with .Values.cosi.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.cosi.restartPolicy }}
{{- if .Values.cosi.affinity }}
@@ -196,8 +195,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -1,9 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.filer.enabled }}
{{- $s3PreStopSeconds := 0 }}
{{- if .Values.filer.s3.enabled }}
{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }}
{{- end }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
@@ -57,7 +53,6 @@ spec:
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.filer.restartPolicy }}
{{- if .Values.filer.affinity }}
@@ -74,7 +69,7 @@ spec:
{{- end }}
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
terminationGracePeriodSeconds: 60
{{- if .Values.filer.priorityClassName }}
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
{{- end }}
@@ -283,9 +278,6 @@ spec:
{{- if .Values.filer.s3.auditLogConfig }}
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
{{- end }}
{{- if gt $s3PreStopSeconds 0 }}
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
{{- end }}
{{- end }}
-master={{ include "seaweedfs.masterServerArg" . }} \
{{- range .Values.filer.extraArgs }}
@@ -453,8 +445,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -48,7 +48,6 @@ spec:
{{- $_ := set $podAnnotations "checksum/master-config" (include "seaweedfs.master.configChecksum" .) }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.master.restartPolicy }}
{{- if .Values.master.affinity }}
@@ -300,8 +299,8 @@ spec:
name: {{ include "seaweedfs.fullname" . }}-master-config
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -1,10 +1,5 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.s3.enabled }}
{{- $preStopSeconds := .Values.s3.preStopSeconds | int }}
{{- $minGracePeriod := add $preStopSeconds 15 }}
{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }}
{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -54,7 +49,6 @@ spec:
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.s3.restartPolicy }}
{{- if .Values.s3.affinity }}
@@ -160,9 +154,6 @@ spec:
{{- if .Values.s3.auditLogConfig }}
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
{{- end }}
{{- if gt $preStopSeconds 0 }}
-preStopSeconds={{ $preStopSeconds }} \
{{- end }}
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \
{{- if .Values.s3.icebergPort }}
-port.iceberg={{ .Values.s3.icebergPort }} \
@@ -289,8 +280,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -43,7 +43,6 @@ spec:
{{- with .Values.sftp.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.sftp.restartPolicy }}
{{- if .Values.sftp.affinity }}
@@ -298,8 +297,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -491,25 +491,6 @@ true
{{- end -}}
{{- end -}}
{{/* True when the volume servers serve HTTPS. Their certificates come from
enableSecurity, so the option fails without it. */}}
{{- define "seaweedfs.volumeHttps" -}}
{{- if ((.Values.global.seaweedfs).volumeHttps).enabled -}}
{{- if not .Values.global.seaweedfs.enableSecurity -}}
{{- fail "global.seaweedfs.volumeHttps.enabled requires global.seaweedfs.enableSecurity, which provides the certificates" -}}
{{- end -}}
true
{{- end -}}
{{- end -}}
{{/* Pod annotation that restarts the readers of security.toml when volume
HTTPS is switched on or off. */}}
{{- define "seaweedfs.volumeHttpsAnnotation" -}}
{{- with include "seaweedfs.volumeHttps" . -}}
checksum/volume-https: {{ sha256sum . | quote }}
{{- end -}}
{{- end -}}
{{/* Read a JWT key from the chart's existing security.toml without fromToml
(which requires Helm >=3.17). Args: (list "<section>" $raw).
Return the single-line TOML string token, including its quotes, so escapes
@@ -286,8 +286,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -1,7 +1,7 @@
{{- include "seaweedfs.compat" . -}}
{{- if include "seaweedfs.securityConfigEnabled" . }}
apiVersion: v1
kind: Secret
kind: ConfigMap
metadata:
name: {{ include "seaweedfs.fullname" . }}-security-config
namespace: {{ include "seaweedfs.namespace" . | quote }}
@@ -11,8 +11,7 @@ metadata:
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
type: Opaque
stringData:
data:
{{- $fullname := include "seaweedfs.fullname" . }}
{{- $securityConfigName := printf "%s-security-config" $fullname }}
{{- $existing := lookup "v1" "ConfigMap" (include "seaweedfs.namespace" .) $securityConfigName }}
@@ -20,11 +19,7 @@ stringData:
{{- $legacyName := printf "%s-%s" (include "seaweedfs.name" .) "security-config" }}
{{- $existing = lookup "v1" "ConfigMap" (include "seaweedfs.namespace" .) $legacyName }}
{{- end }}
{{- $ownConfigMap := eq (dig "metadata" "annotations" "meta.helm.sh/release-name" "" $existing) .Release.Name }}
{{- if and $ownConfigMap (or (and .Values.master.enabled (gt (int .Values.master.updatePartition) 0)) (and .Values.filer.enabled (gt (int .Values.filer.updatePartition) 0))) }}
{{- fail "this upgrade moves security.toml from a ConfigMap to a Secret and removes the ConfigMap; run it with master.updatePartition=0 and filer.updatePartition=0 so no pod is left on the old template" }}
{{- end }}
{{- $existingToml := dig "data" "security.toml" "" (lookup "v1" "Secret" (include "seaweedfs.namespace" .) $securityConfigName) | b64dec | default (dig "data" "security.toml" "" $existing) }}
{{- $existingToml := dig "data" "security.toml" "" $existing }}
{{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
{{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
{{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
@@ -35,7 +30,7 @@ stringData:
# the jwt signing key is read by master and volume server
# the jwt defaults to expire after 10 seconds
[jwt.signing]
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 64 | quote) }}
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
{{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
{{- end }}
@@ -47,7 +42,7 @@ stringData:
# - the Volume server validates the JWT on reading
# the jwt defaults to expire after 60 seconds
[jwt.signing.read]
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 64 | quote) }}
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
{{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
{{- end }}
@@ -59,7 +54,7 @@ stringData:
# - the Filer server validates the JWT on writing
# the jwt defaults to expire after 10 seconds
[jwt.filer_signing]
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 64 | quote) }}
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
{{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
{{- end }}
@@ -71,7 +66,7 @@ stringData:
# - the Filer server validates the JWT on reading
# the jwt defaults to expire after 60 seconds
[jwt.filer_signing.read]
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 64 | quote) }}
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
{{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
{{- end }}
@@ -126,25 +121,13 @@ stringData:
cert = "/usr/local/share/ca-certificates/client/tls.crt"
key = "/usr/local/share/ca-certificates/client/tls.key"
{{- if include "seaweedfs.volumeHttps" . }}
[https.client]
enabled = true
cert = "/usr/local/share/ca-certificates/client/tls.crt"
key = "/usr/local/share/ca-certificates/client/tls.key"
ca = "/usr/local/share/ca-certificates/ca/tls.crt"
[https.volume]
cert = "/usr/local/share/ca-certificates/volume/tls.crt"
key = "/usr/local/share/ca-certificates/volume/tls.key"
{{- else }}
# volume server https: set global.seaweedfs.volumeHttps.enabled
# volume server https options
# Note: work in progress!
# this does not work with other clients, e.g., "weed filer|mount" etc, yet.
[https.client]
enabled = false
[https.volume]
cert = ""
key = ""
{{- end }}
{{- end }}
{{- end }}
@@ -1,5 +1,8 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.vector.enabled }}
{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }}
{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -25,7 +28,6 @@ spec:
annotations:
{{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
{{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }}
@@ -151,8 +153,8 @@ spec:
emptyDir: {}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
{{- range list "ca" "client" "vector" }}
@@ -1,6 +1,6 @@
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
@@ -2,7 +2,7 @@
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
{{- if and $volumeEnabled .Values.volume.ingress.enabled }}
{{- /* Determine service name based on deployment mode */}}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
@@ -1,42 +0,0 @@
{{- /* Ready-only backend for the volume ingress and HTTPRoute; the volume Service publishes not-ready pods for DNS.
Not named volume-client: every volume-<key> name belongs to a group under .Values.volumes. */}}
{{- if and .Values.volume.enabled (not .Values.allInOne.enabled) (or .Values.volume.ingress.enabled .Values.volume.httpRoute.enabled) }}
{{- $name := include "seaweedfs.componentName" (list . "client-volume") }}
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
{{- range $vname, $volume := $volumes }}
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
{{- if and $volume.enabled (eq (include "seaweedfs.componentName" (list $ $volumeName)) $name) }}
{{- fail (printf "The volume Ingress/HTTPRoute Service %q has the same name as the %s Service after truncation. Shorten fullnameOverride or the release name." $name $volumeName) }}
{{- end }}
{{- end }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
namespace: {{ include "seaweedfs.namespace" . | quote }}
labels:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: volume
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
{{- if .Values.volume.annotations }}
annotations:
{{- toYaml .Values.volume.annotations | nindent 4 }}
{{- end }}
spec:
clusterIP: None
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
{{- /* No metrics port, so the volume ServiceMonitor does not scrape each pod twice */}}
ports:
- name: "swfs-volume"
port: {{ .Values.volume.port }}
targetPort: {{ .Values.volume.port }}
protocol: TCP
selector:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: volume
{{- end }}
@@ -23,9 +23,6 @@ metadata:
{{- end }}
spec:
clusterIP: None
{{- /* The master hands out ${POD_NAME}.<this service> as soon as a volume server heartbeats, before it is Ready.
Ingress and HTTPRoute use the client-volume Service instead, which publishes only Ready pods. */}}
publishNotReadyAddresses: true
internalTrafficPolicy: {{ $volume.internalTrafficPolicy | default "Cluster" }}
ports:
- name: "swfs-volume"
@@ -55,7 +55,6 @@ spec:
{{- with $volume.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
{{- if $volume.affinity }}
affinity:
@@ -282,7 +281,7 @@ spec:
httpGet:
path: {{ $volume.readinessProbe.httpGet.path }}
port: {{ $volume.port }}
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.readinessProbe.httpGet.scheme }}{{ end }}
scheme: {{ $volume.readinessProbe.httpGet.scheme }}
initialDelaySeconds: {{ $volume.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ $volume.readinessProbe.periodSeconds }}
successThreshold: {{ $volume.readinessProbe.successThreshold }}
@@ -294,7 +293,7 @@ spec:
httpGet:
path: {{ $volume.livenessProbe.httpGet.path }}
port: {{ $volume.port }}
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.livenessProbe.httpGet.scheme }}{{ end }}
scheme: {{ $volume.livenessProbe.httpGet.scheme }}
initialDelaySeconds: {{ $volume.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ $volume.livenessProbe.periodSeconds }}
successThreshold: {{ $volume.livenessProbe.successThreshold }}
@@ -371,8 +370,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" $ }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" $ }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" $ }}-security-config
{{- end }}
{{- if $.Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
@@ -46,7 +46,6 @@ spec:
{{- with .Values.worker.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.worker.restartPolicy }}
{{- if .Values.worker.affinity }}
@@ -375,8 +374,8 @@ spec:
{{- end }}
{{- if include "seaweedfs.securityConfigEnabled" . }}
- name: security-config
secret:
secretName: {{ include "seaweedfs.fullname" . }}-security-config
configMap:
name: {{ include "seaweedfs.fullname" . }}-security-config
{{- end }}
{{- if .Values.global.seaweedfs.enableSecurity }}
- name: ca-cert
+8 -6
View File
@@ -136,11 +136,13 @@ func TestVectorNetworkPolicyOverrides(t *testing.T) {
}
}
func TestVectorMetricsDefaultImage(t *testing.T) {
output, _ := render(t, "vector.enabled=true", "vector.metricsPort=9329")
for _, want := range []string{"--metrics-port=9329", "path: /ready", "path: /health"} {
if !strings.Contains(output, want) {
t.Fatalf("default image missing %q", want)
}
func TestVectorMetricsImageCompatibility(t *testing.T) {
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm is not installed")
}
out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput()
if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") {
t.Fatalf("incompatible image accepted: %v %s", err, out)
}
render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test")
}
@@ -1,107 +0,0 @@
package tests
import (
"fmt"
"os/exec"
"strings"
"testing"
)
func findDocument(documents []map[string]interface{}, kind, name string) map[string]interface{} {
for _, document := range documents {
if document["kind"] == kind && document["metadata"].(map[string]interface{})["name"] == name {
return document
}
}
return nil
}
func TestVolumeServicePublishesNotReadyPods(t *testing.T) {
_, documents := render(t)
service := findDocument(documents, "Service", "vector-test-seaweedfs-volume")
if service == nil {
t.Fatal("no volume service")
}
if service["spec"].(map[string]interface{})["publishNotReadyAddresses"] != true {
t.Fatal("volume pod DNS names must resolve before the pods are Ready")
}
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
t.Fatal("client-volume service rendered without ingress or httpRoute")
}
}
func TestVolumeIngressUsesReadyOnlyService(t *testing.T) {
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volume.metricsPort=9327")
client := findDocument(documents, "Service", "vector-test-seaweedfs-client-volume")
if client == nil {
t.Fatal("no client-volume service")
}
spec := client["spec"].(map[string]interface{})
if _, found := spec["publishNotReadyAddresses"]; found {
t.Fatal("client-volume must publish only Ready pods")
}
for _, port := range spec["ports"].([]interface{}) {
if port.(map[string]interface{})["name"] == "metrics" {
t.Fatal("client-volume must not expose metrics, or the ServiceMonitor scrapes each pod twice")
}
}
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
if ingress == nil {
t.Fatal("no volume ingress")
}
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
t.Fatalf("volume ingress backend is %v", backend)
}
route := findDocument(documents, "HTTPRoute", "vector-test-seaweedfs-volume")
if route == nil {
t.Fatal("no volume httpRoute")
}
rule := route["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})
if backend := rule["backendRefs"].([]interface{})[0].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
t.Fatalf("volume httpRoute backend is %v", backend)
}
}
func TestVolumeRoutingServiceDoesNotCollideWithVolumeGroups(t *testing.T) {
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volumes.client.replicas=1")
seen := map[string]bool{}
for _, document := range documents {
key := fmt.Sprint(document["kind"], "/", document["metadata"].(map[string]interface{})["name"])
if seen[key] {
t.Fatalf("%s rendered twice", key)
}
seen[key] = true
}
if !seen["Service/vector-test-seaweedfs-volume-client"] || !seen["Service/vector-test-seaweedfs-client-volume"] {
t.Fatal("volumes.client group Service and the routing Service must both render")
}
}
func TestVolumeRoutingServiceRejectsTruncatedCollision(t *testing.T) {
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm is not installed")
}
fullname := strings.Repeat("a", 49) + "-client"
output, err := exec.Command("helm", "template", "test", "..", "--set", "fullnameOverride="+fullname, "--set", "volume.ingress.enabled=true").CombinedOutput()
if err == nil || !strings.Contains(string(output), "same name as the volume Service") {
t.Fatalf("colliding routing Service accepted: %v %s", err, output)
}
}
func TestVolumeIngressAllInOne(t *testing.T) {
_, documents := render(t, "allInOne.enabled=true", "volume.ingress.enabled=true")
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
t.Fatal("client-volume service rendered in all-in-one mode")
}
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
if ingress == nil {
t.Fatal("no volume ingress")
}
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-all-in-one" {
t.Fatalf("all-in-one volume ingress backend is %v", backend)
}
}
+2 -29
View File
@@ -45,18 +45,6 @@ global:
volumeRead: 0
filerWrite: 0
filerRead: 0
# Serve the volume servers' HTTP port over TLS with the chart's volume
# certificate, and make every chart component reach them over HTTPS
# ([https.volume] and [https.client] in security.toml). Requires
# enableSecurity and is not supported with allInOne. Clients outside the
# chart (weed mount, the CSI driver) need their own [https.client].
# The volume servers do not ask for client certificates, because kubelet
# probes cannot present one; JWT signing keeps authorizing volume access.
# [https.client] also sends the admin UI's HTTP lookups on masters and
# S3 over HTTPS, which their plain listeners refuse, so the UI can miss
# volume public URLs and refuse SSE-S3 vector buckets.
volumeHttps:
enabled: false
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
serviceAccountName: "seaweedfs"
serviceAccountAnnotations: {}
@@ -1082,10 +1070,6 @@ filer:
httpsPort: 0
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
# Seconds the gateway keeps serving after SIGTERM while its health checks
# return 503 (see s3.preStopSeconds). Added to the filer pod's 60s
# termination grace period.
preStopSeconds: 0
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
# set to the name of an existing kubernetes Secret with the s3 json config file
@@ -1134,17 +1118,8 @@ s3:
# maxUnavailable: 0
# maxSurge: 25%
updateStrategy: {}
# Time the s3 pod is given to shut down, including any preStop hook.
# With preStopSeconds set, it must be at least preStopSeconds + 15 (the
# in-flight request drain), or the chart refuses to render. A lifecycle
# preStop hook runs before SIGTERM, so add its duration on top; the chart
# cannot tell how long an exec hook takes. preStopSeconds replaces a
# preStop sleep hook.
# Time the s3 pod is given to shut down, including any preStop hook
terminationGracePeriodSeconds: 10
# Seconds the gateway keeps serving after SIGTERM while its health checks
# return 503, so the pod leaves the Service endpoints before it stops
# listening. 0 stops at once.
preStopSeconds: 0
bindAddress: 0.0.0.0
port: 8333
# add additional https port
@@ -1343,8 +1318,7 @@ s3:
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
# the Service endpoints before it receives SIGTERM; keep it shorter than
# terminationGracePeriodSeconds. s3.preStopSeconds does the same from
# inside the gateway and also fails its health checks.
# terminationGracePeriodSeconds.
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
# Example:
# lifecycle:
@@ -2037,7 +2011,6 @@ allInOne:
# The s3-secret.yaml template only reads from .Values.s3.credentials.
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables)
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
# You may specify buckets to be created during the install process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
+1 -3
View File
@@ -45,8 +45,6 @@ Rule of thumb: start with 3 masters, then grow volume servers with data.
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
- Run **3 filers** for HA
- Filers are stateless relative to each other — state lives in the DB
- No quorum among filers: 2 already survive one failure; the third keeps a spare while one is down for upgrades
- Give clients every filer address (e.g. `-filer=f1:8888,f2:8888,f3:8888`) or a load balancer that forwards both the HTTP and gRPC ports (8888 and 18888)
**S3 Servers**
@@ -79,7 +77,7 @@ Keeps housekeeping off the hot path.
|----------------|----------------------|-----------------------------------|
| Master | 3 | Raft quorum, topology, volume map |
| Volume Server | N (scale out) | Stores data needles |
| Filer | 3 (2 minimum) | Metadata gateway, no quorum |
| Filer | 3 | Metadata gateway |
| Metadata DB | HA cluster | Persists filer metadata |
| S3 Server | N (by throughput) | S3 API frontend |
| Admin | 1 | Background job coordinator |
+1084 -1040
View File
File diff suppressed because it is too large. Load diff

Before

Width:  |  Height:  |  Size: 53 KiB

After

Width:  |  Height:  |  Size: 55 KiB

@@ -156,14 +156,11 @@ public class SeaweedWrite {
entry.clearChunks();
entry.addAllChunks(chunks);
FilerProto.CreateEntryResponse response = filerClient.getBlockingStub().createEntry(
filerClient.getBlockingStub().createEntry(
FilerProto.CreateEntryRequest.newBuilder()
.setDirectory(parentDirectory)
.setEntry(entry)
.build());
if (!Strings.isNullOrEmpty(response.getError())) {
throw new IOException(response.getError());
}
}
}
@@ -262,8 +262,6 @@ message CreateEntryRequest {
// Set on a create a filer forwarded to the entry's ring owner, so the owner
// applies it locally instead of forwarding again. Clients leave it unset.
bool is_moved = 8;
// The store write must be durable before the reply, as for ?fsync=true.
bool fsync = 9;
}
// WriteCondition is the precondition the filer evaluates against the existing
+4 -624
View File
@@ -2302,113 +2302,6 @@
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Writable Volumes per Disk",
"description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.",
"type": "timeseries",
"id": 262,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)",
"range": true,
"refId": "A",
"legendFormat": "{{collection}} disks with 2+"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))",
"range": true,
"refId": "B",
"legendFormat": "{{collection}} max on one disk"
}
],
"pluginVersion": "10.3.1"
}
]
},
@@ -3442,102 +3335,6 @@
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Vacuum Dropped Needles",
"description": "Live needles vacuum dropped because their data could not be read back intact (CRC mismatch, malformed or short record), counted since the volume server started. Any non-zero value means a lost replica copy; the volume server log has each dropped key.",
"type": "timeseries",
"id": 267,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 54
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (instance, collection) (SeaweedFS_volumeServer_vacuuming_dropped_needles{cluster=~\"$cluster\"})",
"range": true,
"refId": "A",
"legendFormat": "{{instance}} {{collection}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
@@ -6832,423 +6629,6 @@
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Subscribe Delivery Lag",
"description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope",
"type": "timeseries",
"id": 263,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 71
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "A",
"legendFormat": "p99 {{scope}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "B",
"legendFormat": "p50 {{scope}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Watermark Hold Duration",
"description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)",
"type": "timeseries",
"id": 264,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 71
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "A",
"legendFormat": "p99 {{scope}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
"range": true,
"refId": "B",
"legendFormat": "p50 {{scope}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Delivery Claim / Peer Low-Watermark Age",
"description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled",
"type": "timeseries",
"id": 265,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 79
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})",
"range": true,
"refId": "A",
"legendFormat": "claim {{instance}}"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})",
"range": true,
"refId": "B",
"legendFormat": "peer low {{kind}} {{instance}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"title": "Metadata Subscribers Resumed from Disk",
"description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log",
"type": "timeseries",
"id": 266,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 79
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 4,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"unit": "ops",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))",
"range": true,
"refId": "A",
"legendFormat": "{{scope}} {{instance}}"
}
],
"pluginVersion": "10.3.1"
},
{
"datasource": {
"type": "prometheus",
@@ -7262,7 +6642,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 87
"y": 71
},
"fieldConfig": {
"defaults": {
@@ -7358,7 +6738,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 87
"y": 71
},
"fieldConfig": {
"defaults": {
@@ -7454,7 +6834,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 95
"y": 79
},
"fieldConfig": {
"defaults": {
@@ -7550,7 +6930,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 95
"y": 79
},
"fieldConfig": {
"defaults": {
-6
View File
@@ -7143,10 +7143,8 @@ dependencies = [
"aes",
"aes-gcm",
"anyhow",
"arrow-array",
"arrow-cast",
"arrow-schema",
"arrow-select",
"async-trait",
"base64 0.22.1",
"bytes",
@@ -7160,7 +7158,6 @@ dependencies = [
"hmac 0.12.1",
"http",
"hyper-util",
"jsonb",
"jsonwebtoken",
"lance",
"lance-datafusion",
@@ -7193,7 +7190,6 @@ dependencies = [
"tower",
"tracing",
"tracing-subscriber",
"uuid",
"x509-parser",
"zstd",
]
@@ -7212,7 +7208,6 @@ dependencies = [
"lance",
"lance-index",
"lance-linalg",
"lance-select",
"lance-table",
"object_store",
"roaring",
@@ -7241,7 +7236,6 @@ dependencies = [
"futures",
"lance",
"lance-arrow",
"lance-index",
"lance-linalg",
"moka",
"object_store",
-1
View File
@@ -22,7 +22,6 @@ object_store = "0.13"
lance-index = "10"
lance-linalg = "10"
lance-table = "10"
lance-select = "10"
arrow-array = "58"
arrow-cast = "58"
arrow-schema = "58"
@@ -139,7 +139,6 @@ mod tests {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -168,7 +167,6 @@ mod tests {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let filtered = ann(&dataset, &table, &query).await.unwrap();
let exact = flat(&dataset, &table, &query).await.unwrap();
@@ -206,7 +204,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
+7 -17
View File
@@ -4,7 +4,6 @@
//! compares the elements as typed JSON values: Lance's compares their text, so
//! 1 would match "1".
use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::builder::{BooleanBuilder, StringBuilder};
@@ -20,21 +19,13 @@ use lance::deps::datafusion::logical_expr::{
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
/// Set `filter` on the scanner with the JSONPath functions swapped and the
/// conjuncts on `ready` indexed keys answered by their columns, which it
/// returns.
pub(crate) fn apply(
scanner: &mut Scanner,
filter: &str,
ready: &HashMap<String, String>,
) -> anyhow::Result<Vec<crate::indexed_filter::Indexed>> {
/// Set `filter` on the scanner with the JSONPath functions swapped.
pub(crate) fn apply(scanner: &mut Scanner, filter: &str) -> anyhow::Result<()> {
scanner.filter(filter)?;
let Some(expr) = scanner.get_expr_filter()? else {
return Ok(Vec::new());
};
let (expr, indexed) = crate::indexed_filter::rewrite(expr, ready);
scanner.filter_expr(swap(expr)?);
Ok(indexed)
if let Some(expr) = scanner.get_expr_filter()? {
scanner.filter_expr(swap(expr)?);
}
Ok(())
}
fn swap(expr: Expr) -> Result<Expr> {
@@ -277,7 +268,7 @@ mod tests {
async fn keys(dataset: &Dataset, filter: &str, swapped: bool) -> anyhow::Result<Vec<String>> {
let mut scanner = dataset.scan();
if swapped {
apply(&mut scanner, filter, &HashMap::new())?;
apply(&mut scanner, filter)?;
} else {
scanner.filter(filter)?;
}
@@ -404,7 +395,6 @@ mod tests {
apply(
&mut scanner,
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#,
&HashMap::new(),
)
.unwrap();
let mut swapped = Vec::new();
-78
View File
@@ -104,7 +104,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
}
}
@@ -199,83 +198,6 @@ mod tests {
assert_eq!(filtered.len(), 10);
}
/// A filtered query through the index returns every eligible row when
/// fewer than k match, though the probed partitions cannot hold them all:
/// 400 of 4000 rows match, the index has 63 partitions and probes at most
/// 32 of them.
#[tokio::test]
async fn filtered_ann_returns_every_match_below_k() {
let dir = TempDir::new().unwrap();
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
let tags: Vec<&str> = (0..4000)
.map(|i| if i % 10 == 0 { "rare" } else { "common" })
.collect();
let mut dataset = testing::write_dataset(&dir, &rows, &tags).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
build(
&mut dataset,
&table.vector_column,
INDEX_NAME,
partitions_for(4000),
false,
)
.await
.unwrap();
assert_eq!(partitions_for(4000), 63);
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
let q = Query {
filter: Some("tag = 'rare'".to_string()),
..exact(vector, 500)
};
let flat = query::flat(&dataset, &table, &q).await.unwrap();
assert_eq!(flat.len(), 400);
let ann = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(ann.len(), 400, "the index path dropped eligible rows");
for (f, a) in flat.iter().zip(ann.iter()) {
assert!((f.distance - a.distance).abs() < 1e-5);
}
}
}
/// When no more than k rows match, every match comes back with its own
/// distance, nearest first — not the rows the probes missed with an
/// infinite distance. Ten fragments of 400 rows, 5 of them matching.
#[tokio::test]
async fn filtered_ann_scores_every_match_at_or_below_k() {
let dir = TempDir::new().unwrap();
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
let tags: Vec<&str> = (0..4000)
.map(|i| if i % 800 == 0 { "rare" } else { "common" })
.collect();
let mut dataset = testing::write_dataset(&dir, &rows[..400], &tags[..400]).await;
for chunk in 1..10 {
let r = chunk * 400..(chunk + 1) * 400;
testing::append_dataset(&mut dataset, &rows[r.clone()], &tags[r]).await;
}
let table = VectorTable::resolve(&dataset, None, None).unwrap();
build(&mut dataset, &table.vector_column, INDEX_NAME, 63, false)
.await
.unwrap();
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
for k in [5, 10] {
let q = Query {
filter: Some("tag = 'rare'".to_string()),
..exact(vector, k)
};
let flat = query::flat(&dataset, &table, &q).await.unwrap();
assert_eq!(flat.len(), 5);
let ann = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(ann.len(), 5);
for (f, a) in flat.iter().zip(ann.iter()) {
assert!(
(f.distance - a.distance).abs() < 1e-5,
"k={k}: {flat:?} vs {ann:?}"
);
}
}
}
}
#[tokio::test]
async fn default_probes_take_the_largest_index() {
use lance::index::DatasetIndexExt;
@@ -1,739 +0,0 @@
//! Filters on indexed metadata keys (see `weed_lance_worker::metadata_index`).
//!
//! The gateway renders a string `$eq` or `$in` on a key as an OR of
//! `json_extract(metadata, P) = '"v"'` (or `IN`) and
//! `json_array_contains(metadata, P, '"v"')` over the same values. In the
//! parsed filter, such a top-level conjunct on a ready key is replaced by
//! `array_has` / `array_has_any` on the key's column, which its LABEL_LIST
//! index answers. Everything else, including `$ne` and `$nin`, stays on the
//! JSON path.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use anyhow::Result;
use jsonb::OwnedJsonb;
use lance::Dataset;
use lance::deps::datafusion::common::{Column, ScalarValue};
use lance::deps::datafusion::functions_nested::expr_fn::{array_has, array_has_any};
use lance::deps::datafusion::logical_expr::utils::split_conjunction_owned;
use lance::deps::datafusion::logical_expr::{BinaryExpr, Expr, Operator};
use lance::deps::datafusion::prelude::lit;
use weed_lance_worker::metadata_index::{self, State};
const METADATA_COLUMN: &str = "metadata";
/// The column of each ready indexed key of `dataset`, by metadata key.
pub(crate) fn ready_columns(dataset: &Dataset) -> HashMap<String, String> {
let schema: arrow_schema::Schema = dataset.schema().into();
metadata_index::indexed_keys(&schema)
.into_iter()
.filter(|k| k.state == Some(State::Ready))
.map(|k| (k.key, k.column))
.collect()
}
/// A conjunct a key's column answers: rows whose `column` holds one of
/// `values`.
#[derive(Debug, Clone)]
pub(crate) struct Indexed {
pub column: String,
pub values: Vec<String>,
pub expr: Expr,
}
/// `filter` with each top-level conjunct that is a string `$eq` or `$in` on
/// a key in `ready` answered by the key's column instead, and those
/// replacements.
pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, Vec<Indexed>) {
if ready.is_empty() {
return (filter, Vec::new());
}
let mut indexed: Vec<Indexed> = Vec::new();
let mut rewritten: Option<Expr> = None;
for conjunct in split_conjunction_owned(filter) {
let replacement = string_membership(&conjunct).and_then(|(key, values)| {
let name = ready.get(&key)?;
let column = Expr::Column(Column::from_name(name));
let expr = match values.as_slice() {
[one] => array_has(column, lit(one.as_str())),
_ => {
let values: Vec<ScalarValue> = values
.iter()
.map(|v| ScalarValue::Utf8(Some(v.clone())))
.collect();
let list = ScalarValue::new_list(&values, &arrow_schema::DataType::Utf8, true);
array_has_any(column, Expr::Literal(ScalarValue::List(list), None))
}
};
Some(Indexed {
column: name.clone(),
values,
expr,
})
});
let part = match replacement {
Some(found) => {
let expr = found.expr.clone();
indexed.push(found);
expr
}
None => conjunct,
};
rewritten = Some(and(rewritten, part));
}
(rewritten.unwrap_or(lit(true)), indexed)
}
fn and(left: Option<Expr>, right: Expr) -> Expr {
match left {
Some(left) => Expr::BinaryExpr(BinaryExpr::new(
Box::new(left),
Operator::And,
Box::new(right),
)),
None => right,
}
}
/// The key and string values of a conjunct meaning "the key holds one of
/// these strings, or an array with one of them": the OR of scalar matches
/// and array-member matches over the same values and key.
fn string_membership(expr: &Expr) -> Option<(String, Vec<String>)> {
let mut disjuncts = Vec::new();
split_or(expr, &mut disjuncts);
let mut path: Option<&str> = None;
let mut scalars = BTreeSet::new();
let mut elements = BTreeSet::new();
for disjunct in disjuncts {
let (p, kind) = match unwrap_false_coalesce(disjunct) {
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Eq,
right,
}) => (
json_function(left, "json_extract")?,
Member::Scalar(vec![utf8(right)?]),
),
Expr::InList(list) if !list.negated => (
json_function(&list.expr, "json_extract")?,
Member::Scalar(list.list.iter().map(utf8).collect::<Option<_>>()?),
),
Expr::ScalarFunction(f) if f.func.name() == "json_array_contains" => {
let [data, p, value] = f.args.as_slice() else {
return None;
};
if !is_metadata(data) {
return None;
}
let element: serde_json::Value = serde_json::from_str(utf8(value)?).ok()?;
(utf8(p)?, Member::Element(element.as_str()?.to_string()))
}
_ => return None,
};
if path.is_some_and(|q| q != p) {
return None;
}
path = Some(p);
match kind {
Member::Scalar(values) => scalars.extend(values.into_iter().map(str::to_string)),
Member::Element(value) => {
elements.insert(value);
}
}
}
let key = json_path_key(path?)?;
let rendered: BTreeSet<String> = elements
.iter()
.map(|s| jsonb_text(s))
.collect::<Option<_>>()?;
(!elements.is_empty() && rendered == scalars).then(|| (key, elements.into_iter().collect()))
}
enum Member<'a> {
Scalar(Vec<&'a str>),
Element(String),
}
fn split_or<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) {
match expr {
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::Or,
right,
}) => {
split_or(left, out);
split_or(right, out);
}
Expr::Alias(alias) => split_or(&alias.expr, out),
other => out.push(other),
}
}
/// `x` for `COALESCE(x, false)` or its simplified `x IS NOT NULL AND x`;
/// either is false where `x` is null, which a top-level conjunct treats alike.
fn unwrap_false_coalesce(expr: &Expr) -> &Expr {
match expr {
Expr::ScalarFunction(f) if f.func.name() == "coalesce" => match f.args.as_slice() {
[x, Expr::Literal(ScalarValue::Boolean(Some(false)), _)] => x,
_ => expr,
},
Expr::BinaryExpr(BinaryExpr {
left,
op: Operator::And,
right,
}) => match left.as_ref() {
Expr::IsNotNull(x) if x == right => right,
_ => expr,
},
_ => expr,
}
}
fn utf8(expr: &Expr) -> Option<&str> {
match expr {
Expr::Literal(ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)), _) => {
Some(s.as_str())
}
_ => None,
}
}
fn is_metadata(expr: &Expr) -> bool {
matches!(expr, Expr::Column(c) if c.name == METADATA_COLUMN)
}
/// The JSONPath literal of `name(metadata, path)`.
fn json_function<'a>(expr: &'a Expr, name: &str) -> Option<&'a str> {
match expr {
Expr::ScalarFunction(f) if f.func.name() == name => match f.args.as_slice() {
[data, path] if is_metadata(data) => utf8(path),
_ => None,
},
_ => None,
}
}
/// The key of a `$["key"]` path, with `\\` and `\"` unescaped.
fn json_path_key(path: &str) -> Option<String> {
let inner = path.strip_prefix("$[\"")?.strip_suffix("\"]")?;
let mut key = String::with_capacity(inner.len());
let mut chars = inner.chars();
while let Some(c) = chars.next() {
match c {
'\\' => match chars.next()? {
c @ ('\\' | '"') => key.push(c),
_ => return None,
},
'"' => return None,
c => key.push(c),
}
}
Some(key)
}
/// What json_extract returns for a stored string `s`.
fn jsonb_text(s: &str) -> Option<String> {
let value: OwnedJsonb = serde_json::to_string(s).ok()?.parse().ok()?;
Some(value.to_string())
}
/// Segments searched at once when counting.
const PARALLEL_SEGMENTS: usize = 8;
/// The rows of `scope` (every fragment when None) whose column matches
/// `indexed`, or None once there are more than `limit`. Segments are
/// searched newest first, several at once, and counting stops as soon as
/// the total passes the limit. Rows the index has not reached are counted
/// by a read of the column. Deleted rows an index segment still lists
/// count too: the total is an upper bound, which only ever sends a query
/// to the vector index instead of an exact scan.
pub(crate) async fn count_indexed(
dataset: &Dataset,
indexed: &Indexed,
scope: Option<&[u64]>,
limit: u64,
) -> Result<Option<u64>> {
use futures::{StreamExt, TryStreamExt};
use lance::index::DatasetIndexInternalExt;
use lance_index::metrics::NoOpMetricsCollector;
use lance_index::scalar::{LabelListQuery, SearchResult};
use lance_select::RowSetOps;
let scope: Option<roaring::RoaringBitmap> =
scope.map(|ids| ids.iter().map(|id| *id as u32).collect());
let segments = metadata_index::segments(dataset, &indexed.column).await?;
let query = LabelListQuery::HasAnyLabel(
indexed
.values
.iter()
.map(|v| ScalarValue::Utf8(Some(v.clone())))
.collect(),
);
let mut searches: Vec<futures::future::BoxFuture<'_, Result<u64>>> = Vec::new();
for segment in segments.iter().rev() {
let fragments = match &scope {
Some(scope) => &segment.fragments & scope,
None => segment.fragments.clone(),
};
if fragments.is_empty() {
continue;
}
let uuid = segment.uuid;
let query = &query;
searches.push(Box::pin(async move {
let index = dataset
.open_scalar_index(&indexed.column, &uuid, &NoOpMetricsCollector)
.await?;
let mut found = match index.search(query, &NoOpMetricsCollector).await? {
SearchResult::Exact(rows) | SearchResult::AtMost(rows) => rows.true_rows(),
SearchResult::AtLeast(_) => anyhow::bail!("a LABEL_LIST index answered AtLeast"),
};
found.retain_fragments(fragments.iter());
Ok(found.len().unwrap_or(u64::MAX))
}));
}
let mut counts = futures::stream::iter(searches).buffered(PARALLEL_SEGMENTS);
let mut total = 0u64;
while let Some(count) = counts.try_next().await? {
total = total.saturating_add(count);
if total > limit {
return Ok(None);
}
}
let unindexed: Vec<lance_table::format::Fragment> = dataset
.fragments()
.iter()
.filter(|f| !segments.iter().any(|s| s.fragments.contains(f.id as u32)))
.filter(|f| scope.as_ref().is_none_or(|s| s.contains(f.id as u32)))
.cloned()
.collect();
if !unindexed.is_empty() {
let mut scanner = dataset.scan();
scanner
.filter_expr(indexed.expr.clone())
.with_fragments(unindexed);
total = total.saturating_add(scanner.count_rows().await?);
}
Ok((total <= limit).then_some(total))
}
/// How a filter compares a key.
pub mod operand {
pub const STRING: u8 = 1;
pub const NUMBER: u8 = 2;
pub const BOOL: u8 = 4;
pub const EXISTS: u8 = 8;
}
/// The metadata keys `filter` reads, each with the [`operand`] kinds it is
/// compared with.
pub fn filter_keys(dataset: &Dataset, filter: &str) -> Result<BTreeMap<String, u8>> {
let mut scanner = dataset.scan();
scanner.filter(filter)?;
let mut keys = BTreeMap::new();
if let Some(expr) = scanner.get_expr_filter()? {
collect_keys(&expr, &mut keys);
}
Ok(keys)
}
fn literal_kind(text: &str) -> u8 {
match serde_json::from_str::<serde_json::Value>(text) {
Ok(serde_json::Value::String(_)) => operand::STRING,
Ok(serde_json::Value::Number(_)) => operand::NUMBER,
Ok(serde_json::Value::Bool(_)) => operand::BOOL,
// json_extract's text of a string with control characters is not
// JSON, but it is quoted.
_ if text.starts_with('"') => operand::STRING,
_ => 0,
}
}
fn collect_keys(expr: &Expr, keys: &mut BTreeMap<String, u8>) {
let mut note = |path: &str, kind: u8| {
if let Some(key) = json_path_key(path) {
*keys.entry(key).or_default() |= kind;
}
};
match expr {
Expr::TryCast(cast) if extract(&cast.expr).is_some() => {
note(extract(&cast.expr).unwrap_or_default(), operand::NUMBER);
return;
}
Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
if let (Some(path), Some(value)) = (extract(left), utf8(right)) {
note(path, literal_kind(value));
return;
}
}
Expr::InList(list) => {
if let Some(path) = extract(&list.expr) {
let kind = list
.list
.iter()
.filter_map(utf8)
.fold(0, |k, v| k | literal_kind(v));
note(path, kind);
return;
}
}
Expr::ScalarFunction(f) if is_json_function(f.func.name()) => {
if let [data, path, rest @ ..] = f.args.as_slice()
&& is_metadata(data)
&& let Some(path) = utf8(path)
{
let kind = match (f.func.name(), rest) {
("json_exists", _) => operand::EXISTS,
(_, [value]) => utf8(value).map_or(0, literal_kind),
_ => 0,
};
note(path, kind);
return;
}
}
_ => {}
}
for child in children(expr) {
collect_keys(child, keys);
}
}
fn extract(expr: &Expr) -> Option<&str> {
json_function(expr, "json_extract")
}
fn is_json_function(name: &str) -> bool {
matches!(name, "json_extract" | "json_exists" | "json_array_contains")
}
fn children(expr: &Expr) -> Vec<&Expr> {
use lance::deps::datafusion::common::tree_node::TreeNode;
let mut out = Vec::new();
let _ = expr.apply_children(|child| {
out.push(child);
Ok(lance::deps::datafusion::common::tree_node::TreeNodeRecursion::Continue)
});
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::query::{self, Query};
use crate::table::VectorTable;
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use lance::index::DatasetIndexExt;
use lance_index::IndexType;
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
use lance_linalg::distance::DistanceType;
use std::sync::Arc;
use tempfile::TempDir;
const DIM: usize = 4;
/// The gateway's text for `{"author": "<v>"}`.
fn eq(field: &str, json: &str) -> String {
let json = json.replace('\'', "''");
format!(
"(COALESCE(json_extract(`metadata`, '$[\"{field}\"]') = '{json}', false) OR COALESCE(json_array_contains(`metadata`, '$[\"{field}\"]', '{json}'), false))"
)
}
/// The gateway's text for `{"author": {"$in": [<strings>]}}`.
fn among(values: &[&str]) -> String {
let scalars: Vec<String> = values.iter().map(|v| format!("'\"{v}\"'")).collect();
let mut groups = vec![format!(
"COALESCE(json_extract(`metadata`, '$[\"author\"]') in ({}), false)",
scalars.join(", ")
)];
for v in values {
groups.push(format!(
"COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"{v}\"'), false)"
));
}
format!("({})", groups.join(" OR "))
}
/// Rows `u0..u<n>` by author, plus a few others, in fragments of 100,
/// with the author column ready and indexed and a vector index.
async fn dataset(dir: &TempDir, rows: usize) -> Dataset {
let metadata = Field::new("metadata", DataType::Utf8, true)
.with_metadata([("ARROW:extension:name".to_string(), "arrow.json".to_string())].into());
let schema = Arc::new(Schema::new(vec![
Field::new("key", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
false,
),
metadata,
metadata_index::field("_mk_0", "author", State::Ready),
]));
let metas: Vec<serde_json::Value> = (0..rows)
.map(|i| match i % 10 {
0 => serde_json::json!({"author": "rare"}),
1 => serde_json::json!({"author": ["many", "rare2"]}),
_ => serde_json::json!({"author": "many", "n": i}),
})
.collect();
let batches: Vec<RecordBatch> = (0..rows)
.step_by(100)
.map(|start| {
let range = start..(start + 100).min(rows);
RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(StringArray::from(
range.clone().map(|i| format!("k{i}")).collect::<Vec<_>>(),
)),
Arc::new(FixedSizeListArray::from_iter_primitive::<
arrow_array::types::Float32Type,
_,
_,
>(
range.clone().map(|i| {
Some((0..DIM).map(move |j| Some(((i * 31 + j * 17) % 97) as f32)))
}),
DIM as i32,
)),
Arc::new(StringArray::from(
range
.clone()
.map(|i| Some(metas[i].to_string()))
.collect::<Vec<_>>(),
)),
metadata_index::column_from_json(range.map(|i| Some(&metas[i])), "author"),
],
)
.unwrap()
})
.collect();
let uri = dir.path().to_str().unwrap();
let mut dataset = None::<Dataset>;
for batch in batches {
let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
dataset = Some(match dataset {
None => Dataset::write(reader, uri, None).await.unwrap(),
Some(mut d) => {
d.append(reader, None).await.unwrap();
d
}
});
}
let mut dataset = dataset.unwrap();
dataset
.create_index(
&["_mk_0"],
IndexType::LabelList,
None,
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
true,
)
.await
.unwrap();
crate::index::build(&mut dataset, "vector", crate::index::INDEX_NAME, 4, false)
.await
.unwrap();
dataset
}
fn rewritten(dataset: &Dataset, filter: &str) -> (String, Option<String>) {
let mut scanner = dataset.scan();
let indexed = crate::filter::apply(&mut scanner, filter, &ready_columns(dataset)).unwrap();
(
scanner.get_expr_filter().unwrap().unwrap().to_string(),
indexed.first().map(|i| i.expr.to_string()),
)
}
#[tokio::test]
async fn string_eq_and_in_on_a_ready_key_use_its_column() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 200).await;
assert_eq!(
ready_columns(&dataset),
HashMap::from([("author".to_string(), "_mk_0".to_string())])
);
for (filter, want) in [
(eq("author", "\"rare\""), Some("array_has(_mk_0, Utf8(\"rare\"))")),
(
among(&["rare", "many"]),
Some("array_has_any(_mk_0, List([many, rare]))"),
),
(
format!("{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3", eq("author", "\"rare\"")),
Some("array_has(_mk_0, Utf8(\"rare\"))"),
),
(eq("author", "\"o'neil \\\"q\\\"\""), Some("array_has(_mk_0, Utf8(\"o'neil \"q\"\"))")),
(
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u0001d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
Some("array_has(_mk_0, Utf8(\"c\u{1}d\n\"))"),
),
// The scalar half's text is not what json_extract returns for
// the array half's string, so the two do not agree.
(
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u{1}d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
None,
),
// Not a string, not this key, or not every value on both sides.
(eq("author", "true"), None),
(eq("other", "\"rare\""), None),
(format!("NOT {}", eq("author", "\"rare\"")), None),
(format!("NOT {}", among(&["rare", "many"])), None),
(
"COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"rare\"', false)".to_string(),
None,
),
(
among(&["rare", "many"]).replace("'\"many\"'), false))", "'\"other\"'), false))"),
None,
),
("json_exists(`metadata`, '$[\"author\"]')".to_string(), None),
(format!("({} OR json_exists(`metadata`, '$[\"n\"]'))", eq("author", "\"rare\"")), None),
] {
let (expr, indexed) = rewritten(&dataset, &filter);
assert_eq!(indexed.as_deref(), want, "{filter}");
assert_eq!(expr.contains("json_"), want.is_none() || filter.contains(" AND "), "{filter}: {expr}");
}
}
/// The planner scores few matches exactly and leaves many to the vector
/// index, per scope.
#[tokio::test]
async fn few_indexed_matches_are_scored_exactly() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 1000).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
let query = |filter: &str, fragments: Option<Vec<u64>>| Query {
vector: vec![1.0, 2.0, 3.0, 4.0],
k: 10,
filter: Some(filter.to_string()),
postfilter: false,
distance: DistanceType::L2,
nprobes: None,
index_segments: None,
fragments,
exact_match_limit: None,
};
let rare = eq("author", "\"rare\"");
let many = eq("author", "\"many\"");
let few = |q: Query| {
let (dataset, table) = (&dataset, &table);
async move {
query::few_indexed_matches(dataset, table, &q)
.await
.unwrap()
}
};
// 100 and 900 matches, and 90 in the first two fragments.
assert!(few(query(&rare, None)).await);
assert!(!few(query(&many, None)).await);
let first = Some(vec![0, 1]);
assert!(few(query(&many, first.clone())).await);
assert!(
!few(Query {
exact_match_limit: Some(0),
..query(&rare, None)
})
.await
);
// A JSON-only filter is not counted.
assert!(!few(query("json_exists(`metadata`, '$[\"n\"]')", None)).await);
for (filter, fragments) in [(&rare, None), (&many, first)] {
let q = query(filter, fragments);
let exact = query::flat(&dataset, &table, &q).await.unwrap();
let got = query::ann(&dataset, &table, &q).await.unwrap();
assert_eq!(got.len(), 10);
for (g, e) in got.iter().zip(&exact) {
assert!(
(g.distance - e.distance).abs() < 1e-4,
"{got:?} vs {exact:?}"
);
}
}
}
#[tokio::test]
async fn the_plan_reads_the_scalar_index() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 300).await;
let mut scanner = dataset.scan();
crate::filter::apply(
&mut scanner,
&eq("author", "\"rare\""),
&ready_columns(&dataset),
)
.unwrap();
let plan = scanner.explain_plan(true).await.unwrap();
assert!(plan.contains("LabelList"), "{plan}");
assert!(!plan.contains("json_"), "{plan}");
}
#[tokio::test]
async fn filter_keys_note_how_each_key_is_compared() {
let dir = TempDir::new().unwrap();
let dataset = dataset(&dir, 100).await;
let filter = format!(
"{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3 AND NOT json_exists(`metadata`, '$[\"a.b\\\"c\"]') AND COALESCE(json_extract(`metadata`, '$[\"f\"]') = 'true', false)",
eq("author", "\"rare\"")
);
let keys = filter_keys(&dataset, &filter).unwrap();
assert_eq!(
keys,
BTreeMap::from([
("a.b\"c".to_string(), operand::EXISTS),
("author".to_string(), operand::STRING),
("f".to_string(), operand::BOOL),
("n".to_string(), operand::NUMBER),
])
);
}
/// Counting walks the segments, keeps to the scope and stops once over
/// the limit.
#[tokio::test]
async fn counts_follow_segments_and_scope() {
let dir = TempDir::new().unwrap();
let mut dataset = dataset(&dir, 1000).await;
dataset.drop_index("_mk_0_idx").await.unwrap();
// Ten fragments of 100 rows: four segments of up to 300 rows.
let ids: Vec<u64> = dataset.fragments().iter().map(|f| f.id).collect();
metadata_index::index_fragments(&mut dataset, "_mk_0", 300, 0)
.await
.unwrap();
let segments = metadata_index::segments(&dataset, "_mk_0").await.unwrap();
assert_eq!(segments.len(), 4);
let ready = ready_columns(&dataset);
let indexed =
crate::filter::apply(&mut dataset.scan(), &eq("author", "\"rare\""), &ready).unwrap();
let rare = &indexed[0];
// "rare" in rows i % 10 == 0, "rare2" never matches it.
assert_eq!(
count_indexed(&dataset, rare, None, 1_000).await.unwrap(),
Some(100)
);
assert_eq!(count_indexed(&dataset, rare, None, 99).await.unwrap(), None);
assert_eq!(
count_indexed(&dataset, rare, Some(&ids[..2]), 1_000)
.await
.unwrap(),
Some(20)
);
assert_eq!(
count_indexed(&dataset, rare, Some(&ids[9..]), 1_000)
.await
.unwrap(),
Some(10)
);
assert_eq!(
count_indexed(&dataset, rare, Some(&[]), 1_000)
.await
.unwrap(),
Some(0)
);
}
}
-1
View File
@@ -8,7 +8,6 @@
pub mod coverage;
mod filter;
pub mod index;
pub mod indexed_filter;
pub mod maintenance;
pub mod query;
pub mod segments;
-1
View File
@@ -309,7 +309,6 @@ async fn main() -> Result<()> {
nprobes,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let hits = if flat {
query::flat(&dataset, &vector_table, &query).await?
@@ -312,7 +312,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let mut ids: Vec<u64> = query::flat(&pinned, &resolved, &q)
.await
@@ -735,7 +734,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(query::flat(&dataset, &resolved, &q).await.is_err());
}
+6 -143
View File
@@ -39,15 +39,8 @@ pub struct Query {
/// a worker the unindexed-fragment sweep. Listed ids that no longer exist
/// scan nothing.
pub fragments: Option<Vec<u64>>,
/// Score the matches exactly instead of searching the vector index when
/// the filter's indexed metadata keys match at most this many rows of the
/// scope; 0 never does. None takes [`DEFAULT_EXACT_MATCH_LIMIT`].
pub exact_match_limit: Option<usize>,
}
/// Up to this many matches of an indexed metadata key are scored exactly.
pub const DEFAULT_EXACT_MATCH_LIMIT: usize = 256;
/// One row of a top-K answer.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Hit {
@@ -60,101 +53,14 @@ pub struct Hit {
/// Exact top-K over every fragment, ignoring any index.
pub async fn flat(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
run(scanner(dataset, table, query, false).await?.0, table).await
run(scanner(dataset, table, query, false).await?, table).await
}
/// Top-K with the ANN index where it covers the fragment and a flat pass over
/// the rest. Whether the index is used is Lance's decision per fragment, so
/// rows written after the last index build are always candidates.
///
/// A prefiltered search sees only the probed partitions: it can come back
/// short of k, or with unscored rows (infinite distance) when at most k rows
/// match. Either way it is answered again by an exact scan of the same scope.
///
/// When the filter's indexed metadata keys match few rows of that scope,
/// they are scored exactly to begin with. The decision is per scope, so
/// exact answers of several scopes merge into the exact answer.
pub async fn ann(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
if few_indexed_matches(dataset, table, query).await? {
return exact_in_scope(dataset, table, query).await;
}
let (scanner, indexed) = scanner(dataset, table, query, true).await?;
let hits = run(scanner, table).await?;
if indexed
&& query.filter.is_some()
&& !query.postfilter
&& (hits.len() < query.k || hits.iter().any(|h| !h.distance.is_finite()))
{
return exact_in_scope(dataset, table, query).await;
}
Ok(hits)
}
/// [`flat`] over what [`ann`] searches for `query`: the listed fragments plus
/// the live coverage of the listed index segments, or the whole table.
async fn exact_in_scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
let exact = Query {
index_segments: None,
fragments: scope(dataset, table, query).await?,
..query.clone()
};
run(scanner(dataset, table, &exact, false).await?.0, table).await
}
/// The fragments [`ann`] searches for `query`, or None for the whole table.
async fn scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Option<Vec<u64>>> {
Ok(match &query.index_segments {
Some(segments) if !segments.is_empty() => {
let mut ids = query.fragments.clone().unwrap_or_default();
ids.extend(
crate::segments::covered_fragments(dataset, &table.vector_column, segments).await?,
);
Some(ids)
}
_ => query.fragments.clone(),
})
}
/// Whether the rows of [`ann`]'s scope matching the filter's indexed
/// metadata keys, counted through their scalar indexes, are at most the
/// query's limit. Only a search through the vector index asks.
pub(crate) async fn few_indexed_matches(
dataset: &Dataset,
table: &VectorTable,
query: &Query,
) -> Result<bool> {
let limit = query.exact_match_limit.unwrap_or(DEFAULT_EXACT_MATCH_LIMIT);
let Some(filter) = &query.filter else {
return Ok(false);
};
if limit == 0 || query.postfilter || query.index_segments.as_ref().is_some_and(|s| s.is_empty())
{
return Ok(false);
}
let ready = crate::indexed_filter::ready_columns(dataset);
if ready.is_empty()
|| crate::index::index_names(dataset, &table.vector_column)
.await?
.is_empty()
{
return Ok(false);
}
let indexed = crate::filter::apply(&mut dataset.scan(), filter, &ready)?;
if indexed.is_empty() {
return Ok(false);
}
let scope = scope(dataset, table, query).await?;
// An AND matches at most what its most selective indexed conjunct does.
for conjunct in &indexed {
let count =
crate::indexed_filter::count_indexed(dataset, conjunct, scope.as_deref(), limit as u64)
.await?;
tracing::debug!(column = %conjunct.column, ?count, limit, "indexed metadata filter");
if count.is_some() {
return Ok(true);
}
}
Ok(false)
run(scanner(dataset, table, query, true).await?, table).await
}
/// Fetch rows by key, projected to `columns` (all of them when None).
@@ -195,7 +101,7 @@ async fn scanner(
table: &VectorTable,
query: &Query,
use_index: bool,
) -> Result<(Scanner, bool)> {
) -> Result<Scanner> {
if query.vector.len() != table.dimension {
bail!(
"query dim({}) does not match the table's dim({})",
@@ -211,24 +117,18 @@ async fn scanner(
.use_index(use_index)
.with_row_id();
if let Some(filter) = &query.filter {
let ready = crate::indexed_filter::ready_columns(dataset);
crate::filter::apply(&mut scanner, filter, &ready)?;
crate::filter::apply(&mut scanner, filter)?;
scanner.prefilter(!query.postfilter);
}
// An explicitly empty segment list searches no index, so there is
// nothing to size probes for.
let no_segments = query.index_segments.as_ref().is_some_and(|s| s.is_empty());
let mut indexed = false;
if use_index && !no_segments {
let probes = match query.nprobes {
Some(n) => crate::index::index_names(dataset, &table.vector_column)
.await?
.first()
.map(|_| crate::index::Probes { min: n, max: n }),
Some(n) => Some(crate::index::Probes { min: n, max: n }),
None => crate::index::default_probes(dataset, &table.vector_column).await?,
};
if let Some(probes) = probes {
indexed = true;
// Lance searches `min` partitions, and under a prefilter keeps
// going up to `max` until it has k rows. An open bound would
// make it prepare every partition for that, which costs as much
@@ -278,7 +178,7 @@ async fn scanner(
columns.push(DISTANCE_COLUMN.to_string());
columns.push(ROW_ID_COLUMN.to_string());
scanner.project(&columns)?;
Ok((scanner, indexed))
Ok(scanner)
}
async fn run(scanner: Scanner, table: &VectorTable) -> Result<Vec<Hit>> {
@@ -361,7 +261,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let hits = flat(&dataset, &table, &query).await.unwrap();
assert_eq!(hits.len(), 2);
@@ -403,7 +302,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -483,7 +381,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
@@ -587,7 +484,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(flat(&dataset, &table, &query).await.is_err());
}
@@ -611,7 +507,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let indexed = ann(&dataset, &table, &query).await.unwrap();
let exact = flat(&dataset, &table, &query).await.unwrap();
@@ -620,36 +515,4 @@ mod tests {
exact.iter().map(|h| &h.key).collect::<Vec<_>>()
);
}
/// An explicit probe count does not make an unindexed table's scan count
/// as indexed, so a short filtered answer is not scanned a second time.
#[tokio::test]
async fn explicit_probes_need_an_index() {
let dir = TempDir::new().unwrap();
let rows = crate::testing::vectors(300, DIM);
let mut dataset = write_dataset(&dir, &rows, &["a"; 300]).await;
let table = VectorTable::resolve(&dataset, None, None).unwrap();
let query = Query {
vector: rows[0].clone(),
k: 10,
filter: Some("tag = 'a'".to_string()),
postfilter: false,
distance: DistanceType::L2,
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
};
assert!(!scanner(&dataset, &table, &query, true).await.unwrap().1);
crate::index::build(
&mut dataset,
&table.vector_column,
crate::index::INDEX_NAME,
4,
false,
)
.await
.unwrap();
assert!(scanner(&dataset, &table, &query, true).await.unwrap().1);
}
}
@@ -139,7 +139,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
};
let want: Vec<(u64, u32)> = query::ann(&dataset, &resolved, &base)
.await
@@ -240,7 +239,6 @@ mod tests {
nprobes: None,
index_segments: None,
fragments: Some(vec![u64::MAX]),
exact_match_limit: None,
};
assert!(
query::ann(&dataset, &resolved, &q)
-1
View File
@@ -44,7 +44,6 @@ fn query_for(vector: Vec<f32>, k: usize) -> Query {
nprobes: Some(4),
index_segments: None,
fragments: None,
exact_match_limit: None,
}
}
-1
View File
@@ -46,7 +46,6 @@ tracing.workspace = true
tracing-subscriber.workspace = true
[dev-dependencies]
lance-index = "10"
tempfile = "3"
[build-dependencies]
@@ -1,392 +0,0 @@
//! How often, and how slowly, each index's queries filter on each metadata
//! key over the last hour. Keys come from the parsed filter; the number of
//! indexes and of keys per index is bounded, so arbitrary keys cannot grow
//! memory without limit. Not exported as metrics: a key is no label.
use std::collections::{BTreeMap, HashMap};
use std::sync::Mutex;
use std::time::{Duration, Instant};
use weed_vector_core::indexed_filter::operand;
use crate::pb::vector as pb;
const BUCKETS: usize = 12;
const BUCKET: Duration = Duration::from_secs(300);
pub const WINDOW: Duration = Duration::from_secs(BUCKET.as_secs() * BUCKETS as u64);
pub const MAX_INDEXES: usize = 512;
pub const MAX_KEYS_PER_INDEX: usize = 32;
const MAX_RETIRED_TABLES: usize = 4;
#[derive(Default, Clone, Copy)]
struct Bucket {
/// The bucket's number since the stats started.
at: u64,
queries: u64,
seconds: f64,
/// The operand kinds the bucket's queries compared the key with.
kinds: u8,
}
#[derive(Default)]
struct KeyStats {
buckets: [Bucket; BUCKETS],
}
#[derive(Default)]
struct IndexStats {
/// The table the stats were counted on: an index dropped and created
/// again under its name is another table.
table: String,
/// Tables the name held before, whose late queries are not counted.
retired: Vec<String>,
keys: HashMap<String, KeyStats>,
/// The last bucket a key was left uncounted in.
truncated: Option<u64>,
last: u64,
}
impl IndexStats {
/// Starts counting on `table`, retiring the table counted on before.
fn replace(&mut self, table: &str) {
let mut retired = std::mem::take(&mut self.retired);
if !self.table.is_empty() {
retired.push(std::mem::take(&mut self.table));
if retired.len() > MAX_RETIRED_TABLES {
retired.remove(0);
}
}
*self = IndexStats {
table: table.to_string(),
retired,
last: self.last,
..Default::default()
};
}
}
/// Whether bucket `at` is still in the window of bucket `now`.
fn live(at: u64, now: u64) -> bool {
at + BUCKETS as u64 > now
}
pub struct FilterStats {
start: Instant,
indexes: Mutex<HashMap<String, IndexStats>>,
}
impl Default for FilterStats {
fn default() -> Self {
Self::new()
}
}
impl FilterStats {
pub fn new() -> Self {
Self {
start: Instant::now(),
indexes: Mutex::new(HashMap::new()),
}
}
fn bucket(&self, now: Instant) -> u64 {
(now.duration_since(self.start).as_secs() / BUCKET.as_secs()) + 1
}
/// Drops the stats of `index`, which now holds `table` ("" once
/// dropped): late queries of the table it held before are not counted.
pub fn forget(&self, index: &str, table: &str) {
let mut indexes = self.indexes.lock().unwrap();
if let Some(stats) = indexes.get_mut(index)
&& (table.is_empty() || stats.table != table)
{
stats.replace(table);
}
}
/// One query of `index`, whose table is `table`, that took `elapsed` and
/// filtered on `keys`. Stats counted on another table under the name are
/// dropped; a query of a table the name no longer holds is not counted.
pub fn record(&self, index: &str, table: &str, keys: &BTreeMap<String, u8>, elapsed: Duration) {
self.record_table_at(index, table, keys, elapsed, Instant::now());
}
#[cfg(test)]
fn record_at(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration, now: Instant) {
self.record_table_at(index, "", keys, elapsed, now);
}
fn record_table_at(
&self,
index: &str,
table: &str,
keys: &BTreeMap<String, u8>,
elapsed: Duration,
now: Instant,
) {
if keys.is_empty() {
return;
}
let at = self.bucket(now);
let mut indexes = self.indexes.lock().unwrap();
if !indexes.contains_key(index) && indexes.len() >= MAX_INDEXES {
let Some(oldest) = indexes
.iter()
.min_by_key(|(_, stats)| stats.last)
.map(|(name, _)| name.clone())
else {
return;
};
indexes.remove(&oldest);
}
let stats = indexes.entry(index.to_string()).or_default();
if !table.is_empty() && stats.table != table {
if stats.retired.iter().any(|t| t == table) {
return;
}
if stats.table.is_empty() {
stats.table = table.to_string();
} else {
stats.replace(table);
}
}
stats.last = stats.last.max(at);
for (key, kinds) in keys {
if !stats.keys.contains_key(key) && stats.keys.len() >= MAX_KEYS_PER_INDEX {
// Make room only by dropping a key no query used in the window.
let idle = stats
.keys
.iter()
.find(|(_, k)| k.buckets.iter().all(|b| !live(b.at, at)))
.map(|(name, _)| name.clone());
match idle {
Some(idle) => {
stats.keys.remove(&idle);
}
None => {
stats.truncated = stats.truncated.max(Some(at));
continue;
}
}
}
let entry = stats.keys.entry(key.clone()).or_default();
let bucket = &mut entry.buckets[(at % BUCKETS as u64) as usize];
if bucket.at != at {
*bucket = Bucket {
at,
..Default::default()
};
}
bucket.queries += 1;
bucket.seconds += elapsed.as_secs_f64();
bucket.kinds |= kinds;
}
}
/// The stats of `index` over the window.
pub fn snapshot(&self, index: &str) -> pb::FilterStatsResponse {
self.snapshot_at(index, Instant::now())
}
fn snapshot_at(&self, index: &str, now: Instant) -> pb::FilterStatsResponse {
let at = self.bucket(now);
let window_seconds = now.duration_since(self.start).min(WINDOW).as_secs() as u32;
let indexes = self.indexes.lock().unwrap();
let Some(stats) = indexes.get(index) else {
return pb::FilterStatsResponse {
window_seconds,
..Default::default()
};
};
let mut keys: Vec<pb::FilterKeyStats> = stats
.keys
.iter()
.filter_map(|(key, s)| {
let (queries, seconds, kinds) = s
.buckets
.iter()
.filter(|b| live(b.at, at))
.fold((0, 0.0, 0), |(q, t, k), b| {
(q + b.queries, t + b.seconds, k | b.kinds)
});
(queries > 0).then(|| pb::FilterKeyStats {
key: key.clone(),
queries,
seconds,
string_operand: kinds & operand::STRING != 0,
number_operand: kinds & operand::NUMBER != 0,
bool_operand: kinds & operand::BOOL != 0,
})
})
.collect();
keys.sort_by(|a, b| b.queries.cmp(&a.queries).then(a.key.cmp(&b.key)));
pb::FilterStatsResponse {
keys,
window_seconds,
truncated: stats.truncated.is_some_and(|t| live(t, at)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn keys(pairs: &[(&str, u8)]) -> BTreeMap<String, u8> {
pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
}
#[test]
fn counts_queries_and_latency_per_key_within_the_window() {
let stats = FilterStats::new();
let t0 = stats.start;
let ms = Duration::from_millis;
stats.record_at("b/i", &keys(&[("author", operand::STRING)]), ms(10), t0);
stats.record_at(
"b/i",
&keys(&[("author", operand::STRING), ("year", operand::NUMBER)]),
ms(30),
t0 + Duration::from_secs(600),
);
stats.record_at("b/j", &keys(&[("x", operand::BOOL)]), ms(5), t0);
let got = stats.snapshot_at("b/i", t0 + Duration::from_secs(900));
assert_eq!(got.window_seconds, 900);
assert_eq!(got.keys.len(), 2);
assert_eq!(got.keys[0].key, "author");
assert_eq!(got.keys[0].queries, 2);
assert!((got.keys[0].seconds - 0.04).abs() < 1e-9);
assert!(got.keys[0].string_operand && !got.keys[0].number_operand);
assert_eq!(got.keys[1].key, "year");
assert!(got.keys[1].number_operand);
// The first query leaves the window an hour after its bucket.
let later = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(1));
assert_eq!(later.window_seconds, WINDOW.as_secs() as u32);
assert_eq!(later.keys[0].queries, 1);
let year = later.keys.iter().find(|k| k.key == "year").unwrap();
assert!(year.number_operand);
// A string query that left the window no longer marks the key.
stats.record_at(
"b/i",
&keys(&[("author", operand::NUMBER)]),
ms(1),
t0 + WINDOW + Duration::from_secs(2),
);
let mixed = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(3));
let author = mixed.keys.iter().find(|k| k.key == "author").unwrap();
assert!(author.string_operand && author.number_operand);
let numeric = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(700));
let author = numeric.keys.iter().find(|k| k.key == "author").unwrap();
assert!(!author.string_operand && author.number_operand);
let gone = stats.snapshot_at("b/i", t0 + 2 * WINDOW + Duration::from_secs(900));
assert!(gone.keys.is_empty());
assert!(stats.snapshot_at("other", t0).keys.is_empty());
}
#[test]
fn indexes_and_keys_are_bounded() {
let stats = FilterStats::new();
let t0 = stats.start;
for i in 0..MAX_KEYS_PER_INDEX + 5 {
stats.record_at(
"b/i",
&keys(&[(&format!("k{i}"), operand::STRING)]),
Duration::ZERO,
t0,
);
}
let got = stats.snapshot_at("b/i", t0);
assert_eq!(got.keys.len(), MAX_KEYS_PER_INDEX);
assert!(got.truncated);
// A key idle for the whole window makes room for a new one.
let later = t0 + WINDOW + BUCKET;
stats.record_at(
"b/i",
&keys(&[("new", operand::STRING)]),
Duration::ZERO,
later,
);
let got = stats.snapshot_at("b/i", later);
assert!(!got.truncated, "the uncounted keys left the window");
assert_eq!(got.keys.len(), 1);
assert_eq!(got.keys[0].key, "new");
for i in 0..MAX_INDEXES + 3 {
stats.record_at(
&format!("b/{i}"),
&keys(&[("k", operand::STRING)]),
Duration::ZERO,
later,
);
}
assert_eq!(stats.indexes.lock().unwrap().len(), MAX_INDEXES);
}
#[test]
fn a_record_from_an_earlier_bucket_keeps_the_later_truncation() {
let stats = FilterStats::new();
let t0 = stats.start;
for i in 0..=MAX_KEYS_PER_INDEX {
stats.record_at(
"b/i",
&keys(&[(&format!("k{i}"), operand::STRING)]),
Duration::ZERO,
t0 + BUCKET,
);
}
// Took its time before the lock while the later bucket recorded.
stats.record_at(
"b/i",
&keys(&[("late", operand::STRING)]),
Duration::ZERO,
t0,
);
let got = stats.snapshot_at("b/i", t0 + WINDOW);
assert!(
got.truncated,
"the later bucket's uncounted key is in the window"
);
assert_eq!(stats.indexes.lock().unwrap()["b/i"].last, 2);
}
#[test]
fn stats_follow_the_table_behind_an_index_name() {
let stats = FilterStats::new();
let t0 = stats.start;
let author = keys(&[("author", operand::STRING)]);
let count = |stats: &FilterStats| {
stats
.snapshot_at("b/i", t0)
.keys
.first()
.map_or(0, |k| k.queries)
};
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
assert_eq!(count(&stats), 2);
// The index was created again: its first query starts over.
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
assert_eq!(count(&stats), 1);
// A late query of the old table is not counted on the new one.
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
assert_eq!(count(&stats), 1);
stats.forget("b/i", "");
assert_eq!(count(&stats), 0);
// A first query records its table: a later table starts over.
stats.record_table_at("b/i", "t3", &author, Duration::ZERO, t0);
stats.record_table_at("b/i", "t4", &author, Duration::ZERO, t0);
assert_eq!(count(&stats), 1);
}
#[test]
fn a_late_query_after_a_drop_is_not_counted_on_the_new_table() {
let stats = FilterStats::new();
let t0 = stats.start;
let author = keys(&[("author", operand::STRING)]);
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
stats.forget("b/i", "");
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
assert_eq!(stats.snapshot_at("b/i", t0).keys[0].queries, 2);
}
}
@@ -1,817 +0,0 @@
//! Indexed metadata keys through the VectorService: PutVectors derives the
//! columns, and a query answers the same with or without them.
use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_schema::Schema;
use lance::Dataset;
use lance::dataset::{BatchUDF, NewColumnTransform};
use lance::index::DatasetIndexExt;
use lance_index::IndexType;
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
use tonic::Request;
use weed_lance_worker::metadata_index::{self, STATE_FIELD_META, State};
use weed_vector_core::query::{self, Query};
use weed_vector_core::table::VectorTable;
use crate::pb::vector::{self as pb, vector_service_server::VectorService};
use crate::service::VectorServer;
const DIM: usize = 8;
const ROWS: usize = 3000;
const COLUMN: &str = "_mk_0";
/// How the engine answers a filter once `author` is indexed.
#[derive(Debug, Clone, Copy, PartialEq)]
enum Path {
/// Through the column, at most the exact-match limit of rows: exact.
Exact,
/// Through the column, more rows: the vector index with the column's
/// prefilter.
Ivf,
/// Not through the column.
Json,
}
/// S3 Vectors filters, the gateway's translation of each, and its path.
const FILTERS: [(&str, &str, Path); 22] = [
(
"{\"author\":\"alice\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
Path::Ivf,
),
(
"{\"author\":{\"$eq\":\"bob\"}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
Path::Exact,
),
(
"{\"author\":\"carol\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"carol\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"carol\"'), false))",
Path::Exact,
),
(
"{\"author\":\"x\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
Path::Exact,
),
(
"{\"author\":\"7\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"7\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"7\"'), false))",
Path::Exact,
),
(
"{\"author\":7}",
"(COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) = 7, false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
Path::Json,
),
(
"{\"author\":true}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = 'true', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', 'true'), false))",
Path::Json,
),
(
"{\"author\":{\"$in\":[\"alice\",\"bob\",\"x\"]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"', '\"x\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
Path::Ivf,
),
(
"{\"author\":{\"$in\":[\"alice\",7]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"'), false) OR COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) in (7), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
Path::Json,
),
(
"{\"author\":{\"$ne\":\"alice\"}}",
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$nin\":[\"alice\",\"bob\"]}}",
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$exists\":false}}",
"NOT json_exists(`metadata`, '$[\"author\"]')",
Path::Json,
),
(
"{\"$and\":[{\"author\":\"alice\"},{\"genre\":\"g1\"}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false)) AND (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false)))",
Path::Ivf,
),
(
"{\"$and\":[{\"author\":\"bob\"},{\"year\":{\"$gt\":2005}}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) AND TRY_CAST(json_extract(`metadata`, '$[\"year\"]') AS DOUBLE) > 2005)",
Path::Exact,
),
(
"{\"$or\":[{\"author\":\"bob\"},{\"genre\":\"g2\"}]}",
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) OR (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g2\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g2\"'), false)))",
Path::Json,
),
(
"{\"author\":\"o'neil \\\"q\\\" \\\\ \\n\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"o''neil \\\"q\\\" \\\\ \\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"o''neil \\\"q\\\" \\\\ \\n\"'), false))",
Path::Exact,
),
(
"{\"author\":\"nobody\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"nobody\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"nobody\"'), false))",
Path::Exact,
),
("{\"author\":{\"$in\":[]}}", "false", Path::Json),
(
"{\"genre\":\"g1\"}",
"(COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false))",
Path::Json,
),
(
"{\"author\":\"u5\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"u5\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false))",
Path::Exact,
),
(
"{\"author\":\"ctl\\u0001x\"}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"ctl\\u{1}x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"ctl\\u0001x\"'), false))",
Path::Json,
),
(
"{\"author\":{\"$in\":[\"u5\",\"u6\"]}}",
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"u5\"', '\"u6\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u6\"'), false))",
Path::Exact,
),
];
fn rng(seed: &mut u64) -> f32 {
*seed = seed
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
((*seed >> 33) as f32 / (1u64 << 31) as f32) - 0.5
}
fn vector_of(i: usize) -> Vec<f32> {
let mut centre = (i % 16) as u64 + 1;
let mut noise = i as u64 * 7919 + 13;
(0..DIM)
.map(|_| rng(&mut centre) * 4.0 + rng(&mut noise))
.collect()
}
fn author(i: usize) -> Option<serde_json::Value> {
use serde_json::json;
Some(match i % 100 {
0..=9 => json!("alice"),
10 => json!("bob"),
11 => json!(["carol", "x"]),
12 => json!(7),
13 => json!("7"),
14 => serde_json::Value::Null,
15 => return None,
16 => json!(true),
17 => json!("o'neil \"q\" \\ \n"),
18 => json!("ctl\u{1}x"),
19 => json!(["alice", 3]),
_ => json!(format!("u{}", i % 37)),
})
}
fn row(i: usize, author: Option<serde_json::Value>) -> pb::Vector {
let mut meta = serde_json::json!({"genre": format!("g{}", i % 3), "year": 2000 + i % 10});
if let Some(author) = author {
meta["author"] = author;
}
let serde_json::Value::Object(meta) = meta else {
unreachable!()
};
pb::Vector {
key: format!("k{i}"),
data: vector_of(i),
metadata: Some(crate::service::json_to_struct(meta)),
}
}
struct Fixture {
server: VectorServer,
exact_off: VectorServer,
table: pb::Table,
}
impl Fixture {
async fn new() -> Self {
let bucket = format!(
"mk-{:x}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let table = pb::Table {
id: vec![bucket.clone(), "vector".into(), "x".into()],
location: format!("shared-memory://{bucket}/vector/x"),
..Default::default()
};
let fixture = Self {
server: VectorServer::new().with_schemes(&["shared-memory"]),
exact_off: VectorServer::new()
.with_schemes(&["shared-memory"])
.with_exact_match_limit(Some(0)),
table,
};
fixture
.server
.create_index(Request::new(pb::CreateIndexRequest {
table: Some(fixture.table.clone()),
dimension: DIM as u32,
distance_metric: pb::DistanceMetric::L2 as i32,
..Default::default()
}))
.await
.unwrap();
fixture
}
async fn put(&self, vectors: Vec<pb::Vector>) {
self.server
.put_vectors(Request::new(pb::PutVectorsRequest {
table: Some(self.table.clone()),
vectors,
}))
.await
.unwrap();
}
async fn dataset(&self, version: u64) -> Dataset {
let dataset = Dataset::open(&self.table.location).await.unwrap();
if version == 0 {
dataset
} else {
dataset.checkout_version(version).await.unwrap()
}
}
/// Adds `author`'s column, indexes it and marks it ready, returning the
/// version before it was ready.
async fn index_author(&self) -> u64 {
let mut dataset = self.dataset(0).await;
let output = Arc::new(Schema::new(vec![metadata_index::field(
COLUMN,
"author",
State::Building,
)]));
let schema = output.clone();
dataset
.add_columns(
NewColumnTransform::BatchUDF(BatchUDF {
mapper: Box::new(move |batch: &RecordBatch| {
let column = metadata_index::column_from_metadata(
batch.column_by_name("metadata").unwrap().as_ref(),
"author",
)
.map_err(|e| lance::Error::invalid_input(e.to_string()))?;
Ok(RecordBatch::try_new(schema.clone(), vec![column])?)
}),
output_schema: output,
result_checkpoint: None,
}),
Some(vec!["metadata".to_string()]),
None,
)
.await
.unwrap();
dataset
.create_index(
&[COLUMN],
IndexType::LabelList,
Some(format!("{COLUMN}_idx")),
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
true,
)
.await
.unwrap();
let building = dataset.version().version;
self.set_state(State::Ready).await;
building
}
async fn set_state(&self, state: State) -> u64 {
let mut dataset = self.dataset(0).await;
dataset
.update_field_metadata()
.update(COLUMN, [(STATE_FIELD_META, state.as_str())])
.unwrap()
.await
.unwrap();
dataset.version().version
}
async fn query(
&self,
server: &VectorServer,
version: u64,
filter: &str,
k: u32,
v: &[f32],
) -> Vec<(String, f32)> {
let mut table = self.table.clone();
table.version = version;
server
.query_vectors(Request::new(pb::QueryVectorsRequest {
table: Some(table),
query_vector: v.to_vec(),
top_k: k,
filter: filter.to_string(),
return_distance: true,
..Default::default()
}))
.await
.unwrap()
.into_inner()
.vectors
.into_iter()
.map(|r| (r.key, r.distance))
.collect()
}
/// Exact filtered top-k at `version`.
async fn flat(&self, version: u64, filter: &str, k: u32, v: &[f32]) -> Vec<(String, f32)> {
let dataset = self.dataset(version).await;
let table = VectorTable::resolve(&dataset, None, Some("key")).unwrap();
query::flat(
&dataset,
&table,
&Query {
vector: v.to_vec(),
k: k as usize,
filter: Some(filter.to_string()),
postfilter: false,
distance: lance_linalg::distance::DistanceType::L2,
nprobes: None,
index_segments: None,
fragments: None,
exact_match_limit: None,
},
)
.await
.unwrap()
.into_iter()
.map(|h| (h.key, h.distance))
.collect()
}
/// Every filter answers through the column as through the JSON at
/// `json` (the same rows and indexes, the column not ready), and an
/// exact answer as the exact filtered top-k.
async fn compare(&self, json: u64, ready: u64) {
for q in [7, 1234] {
let v = vector_of(q);
for (s3, filter, path) in FILTERS {
for k in [10, 400] {
let what = format!("{s3} k={k} q={q}");
let exact = self.flat(json, filter, k, &v).await;
assert_same(&self.flat(ready, filter, k, &v).await, &exact, &what);
let json_ann = self.query(&self.server, json, filter, k, &v).await;
let off = self.query(&self.exact_off, ready, filter, k, &v).await;
assert_same(&off, &json_ann, &format!("{what}: indexed prefilter"));
let got = self.query(&self.server, ready, filter, k, &v).await;
let want = if path == Path::Exact {
&exact
} else {
&json_ann
};
assert_same(&got, want, &format!("{what}: {path:?}"));
}
}
}
}
}
fn assert_same(got: &[(String, f32)], want: &[(String, f32)], what: &str) {
assert_eq!(got.len(), want.len(), "{what}: {got:?} vs {want:?}");
// Ties may come back in either order: the same keys, and the same
// distance at each rank.
fn keys(hits: &[(String, f32)]) -> Vec<&str> {
let mut keys: Vec<&str> = hits.iter().map(|(k, _)| k.as_str()).collect();
keys.sort();
keys
}
assert_eq!(keys(got), keys(want), "{what}: {got:?} vs {want:?}");
for (g, w) in got.iter().zip(want) {
let same = (g.1 - w.1).abs() < 1e-4 || (g.1.is_infinite() && w.1.is_infinite());
assert!(same, "{what}: {got:?} vs {want:?}");
if g.0 != w.0 {
let tie = want
.iter()
.any(|(k, d)| k == &g.0 && (d - w.1).abs() < 1e-4);
assert!(
tie,
"{what}: {} at {} is no tie: {got:?} vs {want:?}",
g.0, g.1
);
}
}
}
/// Indexed and JSON answers agree on mixed types, missing keys, arrays,
/// null, untouched operators, and after updates, deletes and rows no
/// scalar or vector index covers yet.
#[tokio::test]
async fn indexed_answers_match_the_json_path() {
let fixture = Fixture::new().await;
for start in (0..ROWS).step_by(300) {
fixture
.put((start..start + 300).map(|i| row(i, author(i))).collect())
.await;
}
let mut dataset = fixture.dataset(0).await;
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
.await
.unwrap();
let json = fixture.index_author().await;
let schema: Schema = fixture.dataset(0).await.schema().into();
assert_eq!(
metadata_index::indexed_keys(&schema)[0].state,
Some(State::Ready)
);
fixture.compare(json, 0).await;
// Updates move rows between values, deletes drop some, and new rows land
// in fragments neither index covers; PutVectors fills the column.
fixture
.put(
(0..60)
.map(|i| row(i, Some(serde_json::json!(["moved", "bob"]))))
.chain((ROWS..ROWS + 200).map(|i| row(i, author(i))))
.collect(),
)
.await;
fixture
.server
.delete_vectors(Request::new(pb::DeleteVectorsRequest {
table: Some(fixture.table.clone()),
keys: (100..130).map(|i| format!("k{i}")).collect(),
}))
.await
.unwrap();
let json = fixture.set_state(State::Building).await;
let ready = fixture.set_state(State::Ready).await;
fixture.compare(json, ready).await;
let moved = r#"(COALESCE(json_extract(`metadata`, '$["author"]') = '"moved"', false) OR COALESCE(json_array_contains(`metadata`, '$["author"]', '"moved"'), false))"#;
let hits = fixture
.query(&fixture.server, ready, moved, 100, &vector_of(0))
.await;
assert_eq!(hits.len(), 60);
}
/// The planner counts and scores per scope: an exact answer over a fragment
/// subset is that subset's exact top-k.
#[tokio::test]
async fn a_scoped_query_is_exact_within_its_fragments() {
let fixture = Fixture::new().await;
for start in (0..ROWS).step_by(300) {
fixture
.put((start..start + 300).map(|i| row(i, author(i))).collect())
.await;
}
let mut dataset = fixture.dataset(0).await;
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
.await
.unwrap();
let json = fixture.index_author().await;
let ready = fixture.dataset(0).await;
let table = VectorTable::resolve(&ready, None, Some("key")).unwrap();
let ids: Vec<u64> = ready.fragments().iter().map(|f| f.id).take(4).collect();
let filter = FILTERS[1].1;
let scoped = || Query {
vector: vector_of(5),
k: 10,
filter: Some(filter.to_string()),
postfilter: false,
distance: lance_linalg::distance::DistanceType::L2,
nprobes: None,
index_segments: None,
fragments: Some(ids.clone()),
exact_match_limit: None,
};
let got = query::ann(&ready, &table, &scoped()).await.unwrap();
let old = fixture.dataset(json).await;
let want = query::flat(&old, &table, &scoped()).await.unwrap();
assert_eq!(want.len(), 10);
let pairs = |hits: Vec<weed_vector_core::Hit>| {
hits.into_iter()
.map(|h| (h.key, h.distance))
.collect::<Vec<_>>()
};
assert_same(&pairs(got), &pairs(want), "scoped bob");
}
/// The engine counts each index's filtered queries by key.
#[tokio::test]
async fn queries_are_counted_by_filter_key() {
let fixture = Fixture::new().await;
fixture
.put((0..50).map(|i| row(i, author(i))).collect())
.await;
for (_, filter, _) in &FILTERS[..3] {
fixture
.query(&fixture.server, 0, filter, 5, &vector_of(1))
.await;
}
let year = FILTERS[13].1;
fixture
.query(&fixture.server, 0, year, 5, &vector_of(1))
.await;
let stats = fixture
.server
.filter_stats(Request::new(pb::FilterStatsRequest {
table: Some(fixture.table.clone()),
}))
.await
.unwrap()
.into_inner();
let author = stats.keys.iter().find(|k| k.key == "author").unwrap();
assert_eq!(author.queries, 4);
assert!(author.string_operand && !author.number_operand);
let year = stats.keys.iter().find(|k| k.key == "year").unwrap();
assert_eq!(year.queries, 1);
assert!(year.number_operand && !year.string_operand);
}
/// A PutVectors that opened the table before its key column was added and
/// commits after: Lance rebases the write and its rows hold NULL in the new
/// column, which the write then fills from the rows' metadata.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_write_that_raced_the_column_add_fills_the_column() {
let fixture = Fixture::new().await;
fixture
.put((0..300).map(|i| row(i, author(i))).collect())
.await;
let unhealed = fixture.dataset(0).await;
let healed = fixture.dataset(0).await;
fixture.index_author().await;
let nulls = |dataset: Dataset| async move {
dataset
.count_rows(Some(format!("{COLUMN} IS NULL")))
.await
.unwrap()
};
let rows: Vec<pb::Vector> = (250..400).map(|i| row(i, author(i))).collect();
let written = crate::service::merge_vectors(unhealed, &rows)
.await
.unwrap();
assert!(written.schema().field(COLUMN).is_some());
assert!(nulls(written).await >= 100, "the race is reproduced");
let written = crate::service::write_vectors(healed, &rows).await.unwrap();
assert_eq!(nulls(written.clone()).await, 0);
let filter = FILTERS[1].1;
let latest = written.version().version;
let v = vector_of(5);
assert_same(
&fixture
.query(&fixture.server, latest, filter, 400, &v)
.await,
&fixture.flat(latest, filter, 400, &v).await,
"bob after the race",
);
assert_eq!(fixture.flat(latest, filter, 400, &v).await.len(), 4);
}
/// A write names no version: one at a version from before a key's column
/// add would commit rows without the column.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_write_at_an_older_version_is_refused() {
let fixture = Fixture::new().await;
fixture
.put((0..10).map(|i| row(i, author(i))).collect())
.await;
let before = fixture.dataset(0).await.version().version;
fixture.index_author().await;
let mut table = fixture.table.clone();
table.version = before;
let refused = fixture
.server
.put_vectors(Request::new(pb::PutVectorsRequest {
table: Some(table),
vectors: (10..20).map(|i| row(i, author(i))).collect(),
}))
.await
.unwrap_err();
assert_eq!(refused.code(), tonic::Code::InvalidArgument);
}
/// Two writes updating the same keys raced the column add, and the first
/// one's fill commits last: each write's rows get the values of its own
/// metadata, not the ones derived before it or from the other write.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_write_that_raced_the_column_add_derives_its_updated_rows() {
let fixture = Fixture::new().await;
fixture
.put((0..300).map(|i| row(i, author(i))).collect())
.await;
let first = fixture.dataset(0).await;
let second = fixture.dataset(0).await;
let written: Schema = first.schema().into();
fixture.index_author().await;
let rows = |name: &str| -> Vec<pb::Vector> {
(250..300)
.map(|i| row(i, Some(serde_json::json!(name))))
.collect()
};
let keys: Vec<String> = (250..300).map(|i| format!("k{i}")).collect();
let first = crate::service::merge_vectors(first, &rows("alice"))
.await
.unwrap();
let second = crate::service::merge_vectors(second, &rows("bob"))
.await
.unwrap();
crate::service::heal_columns(second, &written, &keys)
.await
.unwrap();
crate::service::heal_columns(first, &written, &keys)
.await
.unwrap();
let latest = fixture.dataset(0).await;
let count = |filter: String| {
let latest = latest.clone();
async move { latest.count_rows(Some(filter)).await.unwrap() }
};
assert_eq!(count(format!("{COLUMN} IS NULL")).await, 0);
for (name, filter) in [("alice", FILTERS[0].1), ("bob", FILTERS[1].1)] {
assert_eq!(
count(format!("array_has({COLUMN}, '{name}')")).await,
count(filter.to_string()).await,
"{name}"
);
}
let mut got = fixture
.server
.get_vectors(Request::new(pb::GetVectorsRequest {
table: Some(fixture.table.clone()),
keys: keys.clone(),
return_data: true,
return_metadata: true,
}))
.await
.unwrap()
.into_inner()
.vectors;
got.sort_by_key(|v| v.key[1..].parse::<usize>().unwrap());
assert_eq!(got, rows("bob"));
}
/// Operators ask through the engine; the table records the request and,
/// once the worker has done its part, the key's state, coverage and size.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn keys_are_requested_and_listed_through_the_engine() {
let fixture = Fixture::new().await;
fixture
.put((0..600).map(|i| row(i, author(i))).collect())
.await;
let update = |key: &str, remove: bool| {
fixture
.server
.update_metadata_key(Request::new(pb::UpdateMetadataKeyRequest {
table: Some(fixture.table.clone()),
key: key.to_string(),
remove,
}))
};
let listed = update("author", false).await.unwrap().into_inner();
assert_eq!(listed.keys.len(), 1);
assert_eq!(listed.keys[0].state, "requested");
assert_eq!(listed.keys[0].rows, 600);
// A retry of a committed request answers as the first one did.
let again = update("author", false).await.unwrap().into_inner();
assert_eq!(again.keys.len(), 1);
assert_eq!(again.keys[0].state, "requested");
let mut dataset = fixture.dataset(0).await;
let key = metadata_index::add_column(&mut dataset, "author")
.await
.unwrap();
metadata_index::finish_after(&mut dataset, &key, 1_000_000, std::time::Duration::ZERO)
.await
.unwrap();
let listed = fixture
.server
.metadata_keys(Request::new(pb::MetadataKeysRequest {
table: Some(fixture.table.clone()),
}))
.await
.unwrap()
.into_inner();
assert_eq!((listed.rows, listed.vector_index.as_str()), (600, ""));
let author = &listed.keys[0];
assert_eq!(
(
author.key.as_str(),
author.state.as_str(),
author.column.as_str()
),
("author", "ready", "_mk_0")
);
assert_eq!(
(author.indexed_rows, author.rows, author.segments),
(600, 600, 1)
);
assert!(author.index_bytes > 0);
// Off at once: the next query reads the JSON again.
let listed = update("author", true).await.unwrap().into_inner();
assert_eq!(listed.keys[0].state, "off");
let dataset = fixture.dataset(0).await;
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
// Removing a key that is not indexed needs nothing.
let nobody = update("nobody", true).await.unwrap().into_inner();
assert_eq!(nobody.keys.len(), 1);
assert_eq!(
update("author", false).await.unwrap_err().code(),
tonic::Code::FailedPrecondition,
"being removed"
);
}
/// DeleteIndex purges the table's directory and an index created again
/// under the same name reuses it: the same running engine serves nothing of
/// the old table, neither rows, nor its indexed key, nor its filter stats.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_dropped_and_recreated_index_serves_nothing_old() {
let fixture = Fixture::new().await;
fixture
.put((0..600).map(|i| row(i, author(i))).collect())
.await;
let mut dataset = fixture.dataset(0).await;
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 4, false)
.await
.unwrap();
fixture.index_author().await;
let bob = FILTERS[1].1;
let old = fixture
.query(&fixture.server, 0, bob, 50, &vector_of(10))
.await;
assert_eq!(old.len(), 6);
let server = &fixture.server;
let stats = || {
server.filter_stats(Request::new(pb::FilterStatsRequest {
table: Some(fixture.table.clone()),
}))
};
assert_eq!(stats().await.unwrap().into_inner().keys[0].queries, 1);
// What the catalog's DeleteTable does, then CreateIndex again.
let (store, path) = lance::io::ObjectStore::from_uri(&fixture.table.location)
.await
.unwrap();
store.remove_dir_all(path).await.unwrap();
fixture
.server
.create_index(Request::new(pb::CreateIndexRequest {
table: Some(fixture.table.clone()),
dimension: DIM as u32,
distance_metric: pb::DistanceMetric::L2 as i32,
..Default::default()
}))
.await
.unwrap();
let keys = fixture
.server
.metadata_keys(Request::new(pb::MetadataKeysRequest {
table: Some(fixture.table.clone()),
}))
.await
.unwrap()
.into_inner();
assert!(keys.keys.is_empty());
assert!(
stats().await.unwrap().into_inner().keys.is_empty(),
"the old table's filter stats"
);
fixture
.put((1000..1100).map(|i| row(i, author(i))).collect())
.await;
let new = |hits: &[(String, f32)]| {
hits.iter().all(|(k, _)| {
k[1..]
.parse::<usize>()
.is_ok_and(|i| (1000..1100).contains(&i))
})
};
let unfiltered = fixture
.query(&fixture.server, 0, "", 200, &vector_of(10))
.await;
assert_eq!(unfiltered.len(), 100);
assert!(new(&unfiltered), "{unfiltered:?}");
let bob_now = fixture
.query(&fixture.server, 0, bob, 50, &vector_of(10))
.await;
assert_eq!(bob_now.len(), 1);
assert!(new(&bob_now), "{bob_now:?}");
let author = stats().await.unwrap().into_inner().keys;
assert_eq!((author.len(), author[0].queries), (1, 1));
}
-7
View File
@@ -1,12 +1,5 @@
//! The VectorService gRPC server the S3 gateway talks to.
// PutVectors awaits two Lance merge_insert futures; proving the handler
// future Send takes deeper trait recursion than the default.
#![recursion_limit = "256"]
pub mod filter_stats;
#[cfg(test)]
mod indexed_keys_tests;
pub mod metrics;
pub mod observed;
pub mod read_cache;
-6
View File
@@ -52,11 +52,6 @@ struct Args {
/// tenth of the index's partitions, at least 8; more under a filter.
#[arg(long, env = "WEED_VECTOR_NPROBES")]
nprobes: Option<std::num::NonZeroUsize>,
/// A filtered query whose indexed metadata keys match at most this many
/// rows scores them exactly instead of searching the vector index; 0
/// never does.
#[arg(long, env = "WEED_VECTOR_EXACT_MATCH_LIMIT", default_value_t = weed_vector_core::query::DEFAULT_EXACT_MATCH_LIMIT)]
exact_match_limit: usize,
/// The cluster's security.toml: [grpc.vector] for this server,
/// [grpc.client] for the masters and filers, https.client for volume
/// servers under HTTPS, jwt.signing.read.key for signed volume reads,
@@ -175,7 +170,6 @@ async fn main() -> Result<()> {
.then(|| ReadCache::new(args.read_cache_mb << 20).with_metrics(metrics.clone()));
let mut service = VectorServer::new()
.with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get))
.with_exact_match_limit(Some(args.exact_match_limit))
.with_lance_caches(args.index_cache_mb << 20, args.metadata_cache_mb << 20)
.with_storage(reads);
if let Some(cache) = &cache {
@@ -10,10 +10,6 @@ use tonic_health::{ServingStatus, server::HealthReporter};
use crate::read_cache::ReadCache;
/// The gRPC health service an engine reports serving when its writes derive
/// the indexed metadata key columns.
pub const METADATA_COLUMNS_CAPABILITY: &str = "weed.vector.capability.metadata_columns";
#[derive(Clone)]
pub struct Metrics {
pub registry: Registry,
@@ -124,11 +120,6 @@ impl Metrics {
health
.set_service_status("vector_pb.VectorService", status)
.await;
// A capability, not readiness: the admin refuses to index a
// metadata key while an engine without it could write the table.
health
.set_service_status(METADATA_COLUMNS_CAPABILITY, ServingStatus::Serving)
.await;
}
}
@@ -78,33 +78,6 @@ impl VectorService for ObservedService {
}
result
}
async fn metadata_keys(
&self,
request: Request<pb::MetadataKeysRequest>,
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
let mut in_flight = self.metrics.start("MetadataKeys");
let result = self.inner.metadata_keys(request).await;
in_flight.complete(result.as_ref().err().map(|e| e.code()));
result
}
async fn update_metadata_key(
&self,
request: Request<pb::UpdateMetadataKeyRequest>,
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
let mut in_flight = self.metrics.start("UpdateMetadataKey");
let result = self.inner.update_metadata_key(request).await;
in_flight.complete(result.as_ref().err().map(|e| e.code()));
result
}
async fn filter_stats(
&self,
request: Request<pb::FilterStatsRequest>,
) -> Result<Response<pb::FilterStatsResponse>, Status> {
let mut in_flight = self.metrics.start("FilterStats");
let result = self.inner.filter_stats(request).await;
in_flight.complete(result.as_ref().err().map(|e| e.code()));
result
}
}
#[cfg(test)]
+48 -297
View File
@@ -19,9 +19,7 @@ use weed_lance_worker::catalog::TableDescription;
use weed_lance_worker::dataset;
use weed_lance_worker::direct_read::{DirectReads, Wrappers};
use weed_lance_worker::jobs::vector_index::is_vector_table;
use weed_lance_worker::metadata_index;
use weed_vector_core::index;
use weed_vector_core::indexed_filter;
use weed_vector_core::maintenance::SharedLeases;
use weed_vector_core::query::{self, Query};
use weed_vector_core::table::VectorTable;
@@ -169,13 +167,14 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
};
let vectors_array =
FixedSizeListArray::try_new(item_field.clone(), dimension as i32, values, None)?;
let json: Vec<Option<serde_json::Value>> = vectors
let metas = vectors
.iter()
.map(|v| v.metadata.as_ref().map(struct_to_json))
.collect();
let metas = json
.iter()
.map(|m| m.as_ref().map(serde_json::to_string).transpose())
.map(|v| {
v.metadata
.as_ref()
.map(|m| serde_json::to_string(&struct_to_json(m)))
.transpose()
})
.collect::<Result<Vec<_>, _>>()?;
// The stored column is lance's jsonb (LargeBinary) once the field carries
// the arrow.json extension; encode the strings, else keep the Utf8 batch.
@@ -188,188 +187,11 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
}
_ => Arc::new(StringArray::from(metas)),
};
let (keys, vectors_array): (ArrayRef, ArrayRef) = (Arc::new(keys), Arc::new(vectors_array));
// An indexed metadata key's column is derived from the row's metadata,
// whatever the key's state.
let columns = schema
.fields()
.iter()
.map(|field| match field.name().as_str() {
KEY_COLUMN => Ok(keys.clone()),
VECTOR_COLUMN => Ok(vectors_array.clone()),
METADATA_COLUMN => Ok(metadata_array.clone()),
name => match metadata_index::indexed_key(field) {
Some(indexed) => Ok(metadata_index::column_from_json(
json.iter().map(Option::as_ref),
&indexed.key,
)),
None => bail!("the table has a column {name} that PutVectors cannot fill"),
},
})
.collect::<Result<Vec<_>>>()?;
RecordBatch::try_new(schema, columns).map_err(Into::into)
}
/// Upserts `vectors` by key into `dataset`, the version the request opened,
/// then fills the indexed-key columns a concurrent column add left empty.
pub(crate) async fn write_vectors(
dataset: Dataset,
vectors: &[pb::Vector],
) -> Result<Dataset, Status> {
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
let dataset = merge_vectors(dataset, vectors).await?;
let keys: Vec<String> = vectors.iter().map(|v| v.key.clone()).collect();
heal_columns(dataset, &written, &keys)
.await
.map_err(internal)
}
/// The upsert itself, in the schema of the version `dataset` is at.
pub(crate) async fn merge_vectors(
dataset: Dataset,
vectors: &[pb::Vector],
) -> Result<Dataset, Status> {
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
.map_err(invalid)?
.dimension;
let batch = batch_of(
vectors,
dimension,
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
RecordBatch::try_new(
schema,
vec![Arc::new(keys), Arc::new(vectors_array), metadata_array],
)
.map_err(invalid)?;
// Upsert by key in one merge commit: no window where the old rows are
// already deleted, and a losing writer fails the request instead of
// leaving duplicate keys.
let mut builder = MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
.map_err(internal)?;
builder.when_matched(WhenMatched::UpdateAll);
builder.when_not_matched(WhenNotMatched::InsertAll);
let (dataset, _stats) = builder
.try_build()
.map_err(internal)?
.execute_batches(vec![batch])
.await
.map_err(internal)?;
Ok(Arc::unwrap_or_clone(dataset))
}
/// Attempts [`heal_columns`] makes against writes committing meanwhile.
const HEAL_ATTEMPTS: usize = 10;
/// Lance rebases a write onto a column add committed after the write opened
/// its version: the rows it inserted hold NULL in the new column and the
/// rows it updated keep the value derived from their old metadata. Derive
/// such columns again for the rows of `keys` from their stored metadata.
pub(crate) async fn heal_columns(
mut dataset: Dataset,
written: &Schema,
keys: &[String],
) -> Result<Dataset> {
for _ in 0..HEAL_ATTEMPTS {
let schema: Schema = dataset.schema().into();
let missing: Vec<metadata_index::IndexedKey> = metadata_index::indexed_keys(&schema)
.into_iter()
.filter(|k| written.field_with_name(&k.column).is_err())
.collect();
if missing.is_empty() || keys.is_empty() {
return Ok(dataset);
}
let mut scanner = dataset.scan();
scanner
.filter(&key_list(keys))?
.project(&[KEY_COLUMN, METADATA_COLUMN])?;
let rows: Vec<RecordBatch> = scanner.try_into_stream().await?.try_collect().await?;
let mut fields = vec![schema.field_with_name(KEY_COLUMN)?.clone()];
for k in &missing {
fields.push(schema.field_with_name(&k.column)?.clone());
}
let patch_schema = Arc::new(Schema::new(fields));
let mut patches = Vec::new();
for batch in rows.iter().filter(|b| b.num_rows() > 0) {
let (Some(keys), Some(metadata)) = (
batch.column_by_name(KEY_COLUMN),
batch.column_by_name(METADATA_COLUMN),
) else {
bail!("the scan did not return keys and metadata");
};
let mut columns = vec![keys.clone()];
for k in &missing {
columns.push(metadata_index::column_from_metadata(
metadata.as_ref(),
&k.key,
)?);
}
patches.push(RecordBatch::try_new(patch_schema.clone(), columns)?);
}
if patches.is_empty() {
return Ok(dataset);
}
// Lance's own retry would join these values onto rows a later write
// replaced; derive them again from the version that write left.
let mut builder =
MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec![KEY_COLUMN.to_string()])?;
builder
.when_matched(WhenMatched::UpdateAll)
.when_not_matched(WhenNotMatched::DoNothing)
.conflict_retries(0);
match builder.try_build()?.execute_batches(patches).await {
Ok((healed, _stats)) => {
tracing::info!(
rows = rows.iter().map(|b| b.num_rows()).sum::<usize>(),
"derived indexed metadata key columns a concurrent column add left stale"
);
return Ok(Arc::unwrap_or_clone(healed));
}
Err(lance::Error::TooMuchWriteContention { .. }) => dataset.checkout_latest().await?,
Err(e) => return Err(e.into()),
}
}
bail!("writes kept replacing the rows while their indexed key columns were derived")
}
/// The indexed metadata keys `dataset` records, requested ones first.
async fn metadata_keys(dataset: &Dataset) -> Result<pb::MetadataKeysResponse> {
let rows: u64 = dataset
.fragments()
.iter()
.map(|f| f.num_rows().unwrap_or(0) as u64)
.sum();
let mut keys: Vec<pb::MetadataKey> = metadata_index::requests(dataset)
.into_iter()
.map(|key| pb::MetadataKey {
key,
state: "requested".to_string(),
rows,
..Default::default()
})
.collect();
for key in metadata_index::keys_of(dataset) {
let segments = metadata_index::segments(dataset, &key.column).await?;
keys.push(pb::MetadataKey {
state: key.state.map_or("unknown", |s| s.as_str()).to_string(),
indexed_rows: segments.iter().map(|s| s.rows).sum(),
index_bytes: segments.iter().map(|s| s.bytes).sum(),
segments: segments.len() as u32,
rows,
key: key.key,
column: key.column,
});
}
let vector_index = match index::index_names(dataset, VECTOR_COLUMN).await?.first() {
Some(name) => {
weed_lance_worker::vector_index::index_settings(dataset, name)
.await?
.index_type
}
None => String::new(),
};
Ok(pb::MetadataKeysResponse {
keys,
version: dataset.version().version,
rows,
vector_index,
})
.map_err(Into::into)
}
fn struct_to_json(s: &Struct) -> serde_json::Value {
@@ -408,7 +230,7 @@ fn json_to_value(v: serde_json::Value) -> Value {
Value { kind: Some(kind) }
}
pub(crate) fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
Struct {
fields: map
.into_iter()
@@ -706,8 +528,6 @@ pub struct VectorServer {
lance_cache_bytes: (usize, usize),
leases: SharedLeases,
nprobes: Option<usize>,
exact_match_limit: Option<usize>,
filter_stats: crate::filter_stats::FilterStats,
/// URI schemes a table's location may have.
schemes: &'static [&'static str],
}
@@ -731,8 +551,6 @@ impl VectorServer {
handles: moka::sync::Cache::new(MAX_HANDLES),
leases: SharedLeases::default(),
nprobes: None,
exact_match_limit: None,
filter_stats: Default::default(),
schemes: &["s3"],
}
}
@@ -750,13 +568,6 @@ impl VectorServer {
self
}
/// Matches of a filter's indexed metadata keys scored exactly instead of
/// searched through the vector index, at most; None keeps the default.
pub fn with_exact_match_limit(mut self, limit: Option<usize>) -> Self {
self.exact_match_limit = limit;
self
}
/// Keep tables on the filers and volume servers, not through the gateway.
/// Unset only for tests on an in-process store.
pub fn with_storage(mut self, storage: DirectReads) -> Self {
@@ -929,11 +740,7 @@ impl VectorService for VectorServer {
)
.await;
let dataset = match dataset {
Ok(dataset) => {
self.filter_stats
.forget(&table.id.join("/"), table_id(&dataset));
dataset
}
Ok(dataset) => dataset,
Err(error) => match self.open(&table).await {
Ok(existing)
if matches_index_schema(
@@ -984,7 +791,6 @@ impl VectorService for VectorServer {
store.remove_dir_all(path).await.map_err(internal)?;
}
}
self.filter_stats.forget(&table.id.join("/"), "");
Ok(Response::new(pb::DropIndexResponse {}))
}
@@ -996,37 +802,35 @@ impl VectorService for VectorServer {
let table = req
.table
.ok_or_else(|| Status::invalid_argument("missing table"))?;
if table.version != 0 {
return Err(Status::invalid_argument("a write takes no table version"));
let dataset = self.open(&table).await?;
if req.vectors.is_empty() {
return Ok(Response::new(pb::PutVectorsResponse {
version: dataset.version().version,
}));
}
// A write commits within its deadline of opening the table or not
// at all: a key's column add waits longer than that before the key
// is ready, so a write that opened before the add has committed,
// and been derived again, before any query uses the key.
let merged = tokio::time::timeout(metadata_index::WRITE_DEADLINE, async {
let dataset = self.open(&table).await?;
if req.vectors.is_empty() {
return Ok::<_, Status>((dataset, None));
}
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
Ok((merge_vectors(dataset, &req.vectors).await?, Some(written)))
})
.await
.map_err(|_| {
Status::deadline_exceeded(format!(
"PutVectors took longer than {:?} to commit",
metadata_index::WRITE_DEADLINE
))
})??;
let dataset = match merged {
(dataset, Some(written)) => {
let keys: Vec<String> = req.vectors.iter().map(|v| v.key.clone()).collect();
heal_columns(dataset, &written, &keys)
.await
.map_err(internal)?
}
(dataset, None) => dataset,
};
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
.map_err(invalid)?
.dimension;
let batch = batch_of(
&req.vectors,
dimension,
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
)
.map_err(invalid)?;
// Upsert by key in one merge commit: no window where the old rows are
// already deleted, and a losing writer fails the request instead of
// leaving duplicate keys.
let mut builder =
MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
.map_err(internal)?;
builder.when_matched(WhenMatched::UpdateAll);
builder.when_not_matched(WhenNotMatched::InsertAll);
let (dataset, _stats) = builder
.try_build()
.map_err(internal)?
.execute_batches(vec![batch])
.await
.map_err(internal)?;
Ok(Response::new(pb::PutVectorsResponse {
version: dataset.version().version,
}))
@@ -1104,9 +908,14 @@ impl VectorService for VectorServer {
0 => DEFAULT_PAGE,
n => n.min(MAX_PAGE),
};
// A purge keeps only the latest version, so lease the one this page reads.
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
if segments > 1 {
// Data and metadata come from a second read of this version, so
// lease it as get_vectors does.
let _lease = if columns.len() > 1 {
self.leases.try_pin(&dataset).await.map_err(internal)?
} else {
None
};
let (vectors, next) = list_segment(
&dataset,
&columns,
@@ -1144,7 +953,6 @@ impl VectorService for VectorServer {
&self,
request: Request<pb::QueryVectorsRequest>,
) -> Result<Response<pb::QueryVectorsResponse>, Status> {
let started = std::time::Instant::now();
let req = request.into_inner();
let table = req
.table
@@ -1154,11 +962,6 @@ impl VectorService for VectorServer {
// while this query runs against it.
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
let resolved = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN)).map_err(invalid)?;
let filter_keys = if req.filter.is_empty() {
Default::default()
} else {
indexed_filter::filter_keys(&dataset, &req.filter).unwrap_or_default()
};
if req.query_vector.len() != resolved.dimension {
return Err(Status::invalid_argument(format!(
"query vector carries {} floats, the index dimension is {}",
@@ -1183,7 +986,6 @@ impl VectorService for VectorServer {
nprobes: self.nprobes,
index_segments: None,
fragments: None,
exact_match_limit: self.exact_match_limit,
};
let hits = query::ann(&dataset, &resolved, &query)
.await
@@ -1259,59 +1061,8 @@ impl VectorService for VectorServer {
data: data_map.remove(&hit.key).unwrap_or_default(),
});
}
let index = table.id.join("/");
self.filter_stats
.record(&index, table_id(&dataset), &filter_keys, started.elapsed());
Ok(Response::new(pb::QueryVectorsResponse { vectors }))
}
async fn metadata_keys(
&self,
request: Request<pb::MetadataKeysRequest>,
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
let table = request
.into_inner()
.table
.ok_or_else(|| Status::invalid_argument("missing table"))?;
let dataset = self.open_to_read(&table).await?;
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
Ok(Response::new(
metadata_keys(&dataset).await.map_err(internal)?,
))
}
async fn update_metadata_key(
&self,
request: Request<pb::UpdateMetadataKeyRequest>,
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
let req = request.into_inner();
let table = req
.table
.ok_or_else(|| Status::invalid_argument("missing table"))?;
let mut dataset = self.open(&table).await?;
let updated = if req.remove {
metadata_index::request_removal(&mut dataset, &req.key).await
} else {
metadata_index::request(&mut dataset, &req.key).await
};
updated.map_err(|e| Status::failed_precondition(format!("{e:#}")))?;
Ok(Response::new(
metadata_keys(&dataset).await.map_err(internal)?,
))
}
async fn filter_stats(
&self,
request: Request<pb::FilterStatsRequest>,
) -> Result<Response<pb::FilterStatsResponse>, Status> {
let table = request
.into_inner()
.table
.ok_or_else(|| Status::invalid_argument("missing table"))?;
Ok(Response::new(
self.filter_stats.snapshot(&table.id.join("/")),
))
}
}
#[cfg(test)]
+2 -2
View File
@@ -3195,9 +3195,9 @@ dependencies = [
[[package]]
name = "redb"
version = "4.6.0"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "096b1dd5ec053c9c2a5a26a93aafeadf73ebcb710ea54dc3151f17a8e20535ef"
checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36"
dependencies = [
"libc",
]
+3
View File
@@ -19,6 +19,9 @@ path = "src/main.rs"
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
default = ["5bytes"]
5bytes = []
# Unstable redb cursor bulk-load for full_rebuild. Off in production.
# Pulls redb's experimental_cursor (and therefore experimental-api-5).
redb-experimental-cursor = ["redb/experimental_cursor"]
[lints.clippy]
# Protobuf message literals keep `..Default::default()` on purpose: it is
-1
View File
@@ -147,7 +147,6 @@ service VolumeServer {
message BatchDeleteRequest {
repeated string file_ids = 1;
bool skip_cookie_check = 2;
bool fsync = 3;
}
message BatchDeleteResponse {
+1 -63
View File
@@ -138,8 +138,7 @@ pub struct Cli {
#[arg(long = "concurrentUploadLimitMB", default_value_t = 0)]
pub concurrent_upload_limit_mb: u32,
/// Limit total concurrent download size in MB, 0 means unlimited; reads over
/// the limit are sent to a replica unless readMode is local.
/// Limit total concurrent download size in MB, 0 means unlimited.
#[arg(long = "concurrentDownloadLimitMB", default_value_t = 0)]
pub concurrent_download_limit_mb: u32,
@@ -306,8 +305,6 @@ pub struct VolumeServerConfig {
pub ui_enabled: bool,
pub jwt_signing_key: Vec<u8>,
pub jwt_signing_expires_seconds: i64,
pub jwt_previous_signing_key: Vec<u8>,
pub jwt_previous_read_signing_key: Vec<u8>,
pub jwt_read_signing_key: Vec<u8>,
pub jwt_read_signing_expires_seconds: i64,
pub https_cert_file: String,
@@ -1001,8 +998,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
ui_enabled: sec.jwt_signing_key.is_empty() || sec.access_ui,
jwt_signing_key: sec.jwt_signing_key,
jwt_signing_expires_seconds: sec.jwt_signing_expires,
jwt_previous_signing_key: sec.jwt_previous_signing_key,
jwt_previous_read_signing_key: sec.jwt_previous_read_signing_key,
jwt_read_signing_key: sec.jwt_read_signing_key,
jwt_read_signing_expires_seconds: sec.jwt_read_signing_expires,
https_cert_file: sec.https_cert_file,
@@ -1034,8 +1029,6 @@ fn default_volume_dir() -> String {
pub struct SecurityConfig {
pub jwt_signing_key: Vec<u8>,
pub jwt_signing_expires: i64,
pub jwt_previous_signing_key: Vec<u8>,
pub jwt_previous_read_signing_key: Vec<u8>,
pub jwt_read_signing_key: Vec<u8>,
pub jwt_read_signing_expires: i64,
pub https_cert_file: String,
@@ -1054,8 +1047,6 @@ pub struct SecurityConfig {
pub grpc_volume_allowed_common_names: Vec<String>,
pub tls_policy: TlsPolicy,
pub access_ui: bool,
/// The file was read; otherwise only defaults and WEED_ overrides are set.
pub file_read: bool,
/// IPs from [guard] white_list in security.toml
pub guard_white_list: Vec<String>,
}
@@ -1188,7 +1179,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
match section {
Section::JwtSigningRead => match key {
"key" => cfg.jwt_read_signing_key = value.as_bytes().to_vec(),
"previous_key" => cfg.jwt_previous_read_signing_key = value.as_bytes().to_vec(),
"expires_after_seconds" => {
cfg.jwt_read_signing_expires = value.parse().unwrap_or(60)
}
@@ -1196,7 +1186,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
},
Section::JwtSigning => match key {
"key" => cfg.jwt_signing_key = value.as_bytes().to_vec(),
"previous_key" => cfg.jwt_previous_signing_key = value.as_bytes().to_vec(),
"expires_after_seconds" => {
cfg.jwt_signing_expires = value.parse().unwrap_or(10)
}
@@ -1272,7 +1261,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
// prefix WEED_, uppercase, replace . with _).
// e.g. WEED_JWT_SIGNING_KEY overrides [jwt.signing] key
apply_env_overrides(&mut cfg, env);
cfg.file_read = true;
cfg
}
@@ -1311,12 +1299,6 @@ fn apply_env_overrides(cfg: &mut SecurityConfig, env: EnvLookup<'_>) {
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_EXPIRES_AFTER_SECONDS") {
cfg.jwt_signing_expires = v.parse().unwrap_or(cfg.jwt_signing_expires);
}
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_PREVIOUS_KEY") {
cfg.jwt_previous_signing_key = v.into_bytes();
}
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_READ_PREVIOUS_KEY") {
cfg.jwt_previous_read_signing_key = v.into_bytes();
}
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_READ_KEY") {
cfg.jwt_read_signing_key = v.into_bytes();
}
@@ -2194,50 +2176,6 @@ cipher_suites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
assert_eq!(cfg.jwt_signing_key, b"env-secret");
}
#[test]
fn test_previous_signing_keys_from_file_and_env() {
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(
tmp.path(),
r#"
[jwt.signing]
key = "write-new"
previous_key = "write-old"
[jwt.signing.read]
key = "read-new"
previous_key = "read-old"
"#,
)
.unwrap();
let path = tmp.path().to_str().unwrap();
let cfg = parse_security_config_with_env(path, &fake_env(&[]));
assert_eq!(cfg.jwt_previous_signing_key, b"write-old");
assert_eq!(cfg.jwt_previous_read_signing_key, b"read-old");
let env = fake_env(&[
("WEED_JWT_SIGNING_PREVIOUS_KEY", "env-write-old"),
("WEED_JWT_SIGNING_READ_PREVIOUS_KEY", "env-read-old"),
]);
let cfg = parse_security_config_with_env(path, &env);
assert_eq!(cfg.jwt_previous_signing_key, b"env-write-old");
assert_eq!(cfg.jwt_previous_read_signing_key, b"env-read-old");
}
#[test]
fn test_security_config_file_read() {
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), "[jwt.signing]\nkey = \"secret\"\n").unwrap();
let path = tmp.path().to_str().unwrap().to_string();
assert!(parse_security_config_with_env(&path, &fake_env(&[])).file_read);
drop(tmp);
let env = fake_env(&[("WEED_JWT_SIGNING_KEY", "env-secret")]);
let cfg = parse_security_config_with_env(&path, &env);
assert!(!cfg.file_read);
assert_eq!(cfg.jwt_signing_key, b"env-secret");
}
#[test]
fn test_env_override_takes_precedence_over_file() {
let tmp = tempfile::NamedTempFile::new().unwrap();
+2 -12
View File
@@ -340,7 +340,7 @@ async fn run(
store.set_volume_load_guard(volume_copies_in_flight.clone());
// Build shared state
let mut guard = Guard::new(
let guard = Guard::new(
&config.white_list,
SigningKey(config.jwt_signing_key.clone()),
config.jwt_signing_expires_seconds,
@@ -358,8 +358,6 @@ async fn run(
let security_file = config.security_file.clone();
let cli_white_list = config.white_list.clone();
guard.previous_signing_key = SigningKey(config.jwt_previous_signing_key.clone());
guard.previous_read_signing_key = SigningKey(config.jwt_previous_read_signing_key.clone());
let state = Arc::new(VolumeServerState {
store: RwLock::new(store),
@@ -418,7 +416,6 @@ async fn run(
.to_string_lossy()
.into_owned()
},
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight,
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
@@ -453,7 +450,6 @@ async fn run(
{
let mut store = state.store.write().unwrap();
store.store_dirs = config.folders.clone();
for (i, dir) in config.folders.iter().enumerate() {
let idx_dir = if config.idx_folder.is_empty() {
dir.as_str()
@@ -621,13 +617,9 @@ async fn run(
}
// 2. Reload security config (Go's Reload)
'security: {
{
info!("Reloading security config...");
let sec = config::parse_security_config(&state_reload.security_file);
if !sec.file_read {
warn!("SIGHUP: security config cannot be read, keeping the keys in effect");
break 'security;
}
let mut whitelist = state_reload.cli_white_list.clone();
whitelist.extend(sec.guard_white_list.iter().cloned());
let mut guard = state_reload.guard.write().unwrap();
@@ -637,8 +629,6 @@ async fn run(
sec.jwt_signing_expires,
SigningKey(sec.jwt_read_signing_key),
sec.jwt_read_signing_expires,
SigningKey(sec.jwt_previous_signing_key),
SigningKey(sec.jwt_previous_read_signing_key),
);
}
-14
View File
@@ -85,19 +85,6 @@ pub static VACUUMING_COMMIT_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(||
.expect("metric can be created")
});
/// Live needles vacuum dropped because their data could not be read back
/// intact, with label `collection`.
pub static VACUUMING_DROPPED_NEEDLES_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_vacuuming_dropped_needles",
"Counter of live needles dropped by vacuum because their data could not be read back intact.",
),
&["collection"],
)
.expect("metric can be created")
});
/// Vacuuming duration histogram with label `type` (compact/commit).
pub static VACUUMING_HISTOGRAM: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
@@ -415,7 +402,6 @@ pub fn register_metrics() {
Box::new(VACUUMING_COMPACT_COUNTER.clone()),
Box::new(VACUUMING_COMMIT_COUNTER.clone()),
Box::new(VACUUMING_HISTOGRAM.clone()),
Box::new(VACUUMING_DROPPED_NEEDLES_COUNTER.clone()),
Box::new(VOLUME_GAUGE.clone()),
Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
Box::new(MAX_VOLUMES.clone()),
+23 -207
View File
@@ -83,13 +83,7 @@ pub fn gen_jwt(
}
/// Decode and validate a JWT token.
/// A token whose signature does not match `signing_key` is checked against
/// each non-empty `previous_keys` entry, like Go's `security.DecodeJwt`.
pub fn decode_jwt(
signing_key: &SigningKey,
token: &str,
previous_keys: &[&SigningKey],
) -> Result<FileIdClaims, JwtError> {
pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims, JwtError> {
if signing_key.is_empty() {
return Err(JwtError::NoSigningKey);
}
@@ -103,64 +97,13 @@ pub fn decode_jwt(
validation.validate_nbf = true;
validation.leeway = 0;
let mut result = decode::<FileIdClaims>(
let data = decode::<FileIdClaims>(
token,
&DecodingKey::from_secret(&signing_key.0),
&validation,
);
for previous in previous_keys {
let bad_signature = matches!(&result, Err(e)
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::InvalidSignature));
if !bad_signature {
break;
}
if previous.is_empty() || previous.0 == signing_key.0 {
continue;
}
result = decode::<FileIdClaims>(token, &DecodingKey::from_secret(&previous.0), &validation);
}
)?;
Ok(result?.claims)
}
/// Reports a signed, fid-bound write token whose only failure is expiry. It
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
pub fn is_expired_volume_jwt(
signing_key: &SigningKey,
token: &str,
expected_fid: &str,
previous_keys: &[&SigningKey],
) -> bool {
match decode_jwt(signing_key, token, previous_keys) {
Err(JwtError::Jwt(e))
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
_ => return false,
}
#[derive(Deserialize)]
struct ExpiredClaims {
fid: Option<String>,
iat: Option<u64>,
}
let mut validation = Validation::new(Algorithm::HS256);
validation.required_spec_claims.clear();
validation.validate_exp = false;
validation.validate_nbf = true;
validation.leeway = 0;
let Some(data) = std::iter::once(signing_key)
.chain(previous_keys.iter().copied())
.filter(|key| !key.is_empty())
.find_map(|key| {
decode::<ExpiredClaims>(token, &DecodingKey::from_secret(&key.0), &validation).ok()
})
else {
return false;
};
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
Ok(data.claims)
}
// ============================================================================
@@ -175,9 +118,6 @@ pub struct Guard {
pub expires_after_sec: i64,
pub read_signing_key: SigningKey,
pub read_expires_after_sec: i64,
/// Accepted when verifying, never used to sign.
pub previous_signing_key: SigningKey,
pub previous_read_signing_key: SigningKey,
/// Combined flag: true when whitelist is non-empty OR signing key is present.
/// Matches Go's `isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0`.
is_write_active: bool,
@@ -198,8 +138,6 @@ impl Guard {
expires_after_sec,
read_signing_key,
read_expires_after_sec,
previous_signing_key: SigningKey(vec![]),
previous_read_signing_key: SigningKey(vec![]),
is_write_active: false,
};
guard.update_whitelist(whitelist);
@@ -244,15 +182,11 @@ impl Guard {
expires_after_sec: i64,
read_signing_key: SigningKey,
read_expires_after_sec: i64,
previous_signing_key: SigningKey,
previous_read_signing_key: SigningKey,
) {
self.signing_key = signing_key;
self.expires_after_sec = expires_after_sec;
self.read_signing_key = read_signing_key;
self.read_expires_after_sec = read_expires_after_sec;
self.previous_signing_key = previous_signing_key;
self.previous_read_signing_key = previous_read_signing_key;
let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
@@ -295,16 +229,6 @@ impl Guard {
false
}
/// The write (or read) signing key and the previous key still accepted for
/// it. Mirrors Go's `Guard.VerificationKeys`.
pub fn verification_keys(&self, is_write: bool) -> (&SigningKey, &SigningKey) {
if is_write {
(&self.signing_key, &self.previous_signing_key)
} else {
(&self.read_signing_key, &self.previous_read_signing_key)
}
}
/// Check if a read signing key is configured.
pub fn has_read_signing_key(&self) -> bool {
!self.read_signing_key.is_empty()
@@ -314,14 +238,18 @@ impl Guard {
/// `is_write` determines which signing key to use.
/// Returns Ok(()) if valid, or if security is disabled.
pub fn check_jwt(&self, token: Option<&str>, is_write: bool) -> Result<(), JwtError> {
let (key, previous) = self.verification_keys(is_write);
let key = if is_write {
&self.signing_key
} else {
&self.read_signing_key
};
if key.is_empty() {
return Ok(()); // Security disabled for this operation type
}
let token = token.ok_or(JwtError::MissingToken)?;
decode_jwt(key, token, &[previous])?;
decode_jwt(key, token)?;
Ok(())
}
@@ -332,14 +260,18 @@ impl Guard {
expected_fid: &str,
is_write: bool,
) -> Result<(), JwtError> {
let (key, previous) = self.verification_keys(is_write);
let key = if is_write {
&self.signing_key
} else {
&self.read_signing_key
};
if key.is_empty() {
return Ok(());
}
let token = token.ok_or(JwtError::MissingToken)?;
let claims = decode_jwt(key, token, &[previous])?;
let claims = decode_jwt(key, token)?;
match claims.fid {
None => {
@@ -453,7 +385,7 @@ mod tests {
fn test_jwt_round_trip() {
let key = SigningKey::from_string("test-secret-key");
let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
let claims = decode_jwt(&key, &token, &[]).unwrap();
let claims = decode_jwt(&key, &token).unwrap();
assert_eq!(claims.fid, Some("3,01637037d6".to_string()));
}
@@ -466,7 +398,7 @@ mod tests {
#[test]
fn test_jwt_invalid_token() {
let key = SigningKey::from_string("secret");
let result = decode_jwt(&key, "invalid.token.here", &[]);
let result = decode_jwt(&key, "invalid.token.here");
assert!(result.is_err());
}
@@ -475,60 +407,7 @@ mod tests {
let key1 = SigningKey::from_string("secret1");
let key2 = SigningKey::from_string("secret2");
let token = gen_jwt(&key1, 3600, "1,abc").unwrap();
assert!(decode_jwt(&key2, &token, &[]).is_err());
}
#[test]
fn test_is_expired_volume_jwt_classifies_expiry_only() {
let key = SigningKey::from_string("write-key");
let fid = "3,01637037d6";
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let sign = |claims: serde_json::Value, key: &SigningKey| {
encode(
&Header::new(Algorithm::HS256),
&claims,
&EncodingKey::from_secret(&key.0),
)
.unwrap()
};
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
assert!(is_expired_volume_jwt(&key, &expired, fid, &[]));
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7", &[]));
assert!(!is_expired_volume_jwt(
&SigningKey::from_string("other-key"),
&expired,
fid,
&[]
));
assert!(!is_expired_volume_jwt(
&SigningKey(vec![]),
&expired,
fid,
&[]
));
let immature = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &immature, fid, &[]));
let future_issued = sign(
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
&key,
);
assert!(!is_expired_volume_jwt(&key, &future_issued, fid, &[]));
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
assert!(!is_expired_volume_jwt(&key, &live, fid, &[]));
let unsigned = format!(
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
base64::Engine::encode(
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
)
);
assert!(!is_expired_volume_jwt(&key, &unsigned, fid, &[]));
assert!(decode_jwt(&key2, &token).is_err());
}
#[test]
@@ -628,88 +507,25 @@ mod tests {
11,
SigningKey::from_string("new-read"),
61,
SigningKey(vec![]),
SigningKey(vec![]),
);
let token = gen_jwt(&guard.signing_key, 60, "3,01637037d6").unwrap();
// Validates with the rotated key, not the old one.
assert!(decode_jwt(&guard.signing_key, &token, &[]).is_ok());
assert!(decode_jwt(&SigningKey::from_string("old-write"), &token, &[]).is_err());
assert!(decode_jwt(&guard.signing_key, &token).is_ok());
assert!(decode_jwt(&SigningKey::from_string("old-write"), &token).is_err());
assert_eq!(guard.expires_after_sec, 11);
assert_eq!(guard.read_expires_after_sec, 61);
}
#[test]
fn test_previous_signing_key_is_accepted() {
let fid = "3,01637037d6";
let accepts = |guard: &Guard, key: &str, is_write: bool| {
let token = gen_jwt(&SigningKey::from_string(key), 60, fid).unwrap();
guard
.check_jwt_for_file(Some(&token), fid, is_write)
.is_ok()
};
let mut guard = Guard::new(
&[],
SigningKey::from_string("write-new"),
10,
SigningKey::from_string("read-new"),
60,
);
guard.previous_signing_key = SigningKey::from_string("write-old");
guard.previous_read_signing_key = SigningKey::from_string("read-old");
assert!(accepts(&guard, "write-new", true));
assert!(accepts(&guard, "write-old", true));
assert!(accepts(&guard, "read-old", false));
assert!(!accepts(&guard, "other", true));
assert!(!accepts(&guard, "read-old", true));
let expired = encode(
&Header::new(Algorithm::HS256),
&serde_json::json!({"fid": fid, "exp": 1}),
&EncodingKey::from_secret(b"write-old"),
)
.unwrap();
let (key, previous) = guard.verification_keys(true);
assert!(is_expired_volume_jwt(key, &expired, fid, &[previous]));
assert!(!is_expired_volume_jwt(key, &expired, fid, &[]));
guard.update_signing_keys(
SigningKey::from_string("write-new"),
10,
SigningKey::from_string("read-new"),
60,
SigningKey(vec![]),
SigningKey(vec![]),
);
assert!(!accepts(&guard, "write-old", true));
assert!(!accepts(&guard, "read-old", false));
}
#[test]
fn test_update_signing_keys_toggles_write_active() {
let mut guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
assert!(guard.check_jwt(None, true).is_ok());
guard.update_signing_keys(
SigningKey::from_string("write"),
10,
SigningKey(vec![]),
0,
SigningKey(vec![]),
SigningKey(vec![]),
);
guard.update_signing_keys(SigningKey::from_string("write"), 10, SigningKey(vec![]), 0);
assert!(guard.is_write_active);
guard.update_signing_keys(
SigningKey(vec![]),
0,
SigningKey(vec![]),
0,
SigningKey(vec![]),
SigningKey(vec![]),
);
guard.update_signing_keys(SigningKey(vec![]), 0, SigningKey(vec![]), 0);
assert!(!guard.is_write_active);
}
+2 -15
View File
@@ -110,17 +110,6 @@ pub fn load_outgoing_grpc_tls(
}))
}
#[cfg(test)]
impl OutgoingGrpcTlsConfig {
pub(crate) fn self_signed_for_test() -> Self {
Self {
cert_pem: tests::TEST_CERT_PEM.to_string(),
key_pem: tests::TEST_KEY_PEM.to_string(),
ca_pem: tests::TEST_CERT_PEM.to_string(),
}
}
}
pub fn grpc_endpoint_uri(grpc_host_port: &str, tls: Option<&OutgoingGrpcTlsConfig>) -> String {
let scheme = if tls.is_some() { "https" } else { "http" };
format!("{}://{}", scheme, grpc_host_port)
@@ -371,8 +360,8 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::time::Duration;
pub(super) const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
pub(super) const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
#[test]
fn test_build_grpc_endpoint_with_tls_resolves_crypto_provider() {
@@ -436,8 +425,6 @@ mod tests {
ui_enabled: false,
jwt_signing_key: vec![],
jwt_signing_expires_seconds: 0,
jwt_previous_signing_key: vec![],
jwt_previous_read_signing_key: vec![],
jwt_read_signing_key: vec![],
jwt_read_signing_expires_seconds: 0,
https_cert_file: String::new(),
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
-1
View File
@@ -1698,7 +1698,6 @@ mod tests {
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
-2
View File
@@ -156,8 +156,6 @@ mod tests {
ui_enabled: false,
jwt_signing_key: vec![],
jwt_signing_expires_seconds: 0,
jwt_previous_signing_key: vec![],
jwt_previous_read_signing_key: vec![],
jwt_read_signing_key: vec![],
jwt_read_signing_expires_seconds: 0,
https_cert_file: String::new(),
+9 -31
View File
@@ -2384,45 +2384,23 @@ pub(crate) async fn recover_missing_ec_indexes(
"cannot look up peers to recover missing .ecx: {}",
e
);
Vec::new()
continue;
}
};
// Under the volume's lifecycle lock, so an in-flight VolumeEcShardsCopy
// of it, which may unlink the shards it renamed in, never has them
// mounted from under it.
let _lifecycle = state.ec_lifecycle_lock(m.vid).lock_owned().await;
let current = state
.store
.read()
.unwrap()
.collect_ec_volumes_missing_index_only(Some(m.vid))
.into_iter()
.find(|x| x.vid == m.vid && x.collection == m.collection);
// A copy holding the lock may have installed the index meanwhile: skip
// the fetch, but its shards still need the mount.
if let Some(current) = &current {
if peers.is_empty() {
tracing::warn!(
volume_id = m.vid.0,
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
);
continue;
}
if !fetch_ec_index_from_peers(state, current, &peers).await {
continue;
}
if peers.is_empty() {
tracing::warn!(
volume_id = m.vid.0,
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
);
continue;
}
state
.store
.write()
.unwrap()
.mount_recovered_ec_shards_only(Some(m.vid));
if current.is_some() {
if fetch_ec_index_from_peers(state, m, &peers).await {
recovered += 1;
}
}
if recovered > 0 {
state.store.write().unwrap().mount_recovered_ec_shards();
tracing::info!(
"recovered missing EC index for {} volume(s) from peers and mounted their shards",
recovered
@@ -125,11 +125,6 @@ pub struct VolumeServerState {
pub cli_white_list: Vec<String>,
/// Path to state.pb file for persisting VolumeServerState across restarts.
pub state_file_path: String,
/// Per-vid EC lifecycle lock, serializing EC ops that install or remove a
/// vid's files. A tokio Mutex so the guard survives the copy's `.await` IO.
pub ec_lifecycle_locks: std::sync::Mutex<
std::collections::HashMap<crate::storage::types::VolumeId, Arc<tokio::sync::Mutex<()>>>,
>,
/// Volumes with an EC decode in flight. A dropped request leaves the
/// blocking job running; this keeps a retry from racing it on the
/// same volume files.
@@ -156,20 +151,6 @@ pub struct VolumeServerState {
}
impl VolumeServerState {
/// Return the per-vid EC lifecycle lock, creating it on first use. Entries
/// are never reaped.
pub fn ec_lifecycle_lock(
&self,
vid: crate::storage::types::VolumeId,
) -> Arc<tokio::sync::Mutex<()>> {
self.ec_lifecycle_locks
.lock()
.unwrap()
.entry(vid)
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
/// The master lookups should go to: the leader the heartbeat confirmed,
/// or empty while none is known, in its announced `host:port[.grpcPort]`
/// form. Mirrors Go's `vs.getCurrentMaster()`.
-1
View File
@@ -467,7 +467,6 @@ mod tests {
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
+4 -57
View File
@@ -58,8 +58,6 @@ pub struct DiskLocation {
/// skips a claimed vid outright and claims any other through its load,
/// so a VolumeCopy's .note sweep cannot unlink files it still writes.
pub(crate) load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
/// The data directory of every disk on this server.
pub(crate) store_dirs: Vec<String>,
}
/// Keeps `vid` in the copy-claim set through one scan load so a VolumeCopy
@@ -125,7 +123,6 @@ impl DiskLocation {
disk_free_bytes: AtomicU64::new(0),
min_free_space,
load_guard: None,
store_dirs: Vec::new(),
})
}
@@ -489,20 +486,8 @@ impl DiskLocation {
// Custom ratio comes from the volume's own .vif; the server holds no
// cluster EC config in memory.
let data_shards = ec_data_shards_from_vif(
&self.directory,
&self.idx_directory,
&self.store_dirs,
collection,
vid,
);
if data_shards == 0 {
warn!(
volume_id = vid.0,
"sibling .vif files disagree on the encode; keeping EC shards"
);
return true;
}
let data_shards =
ec_data_shards_from_vif(&self.directory, &self.idx_directory, collection, vid);
// On-disk .dat: an empty <= superblock one is a stub; a transient stat
// error keeps the shards rather than deleting.
@@ -829,24 +814,6 @@ impl DiskLocation {
self.volumes.len()
}
/// Number of local volumes of `collection` that still take writes: not
/// read-only and, when `volume_size_limit` is known (non zero), with a .dat
/// below it. The .dat size is what the master compares with the limit to
/// stop writes. Mirrors Go's WritableCollectionVolumesLen.
pub fn writable_collection_volumes_len(
&self,
collection: &str,
volume_size_limit: u64,
) -> usize {
self.volumes
.values()
.filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only())
.filter(|v| {
volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit
})
.count()
}
/// Get all volume IDs, sorted.
pub fn volume_ids(&self) -> Vec<VolumeId> {
let mut ids: Vec<VolumeId> = self.volumes.keys().copied().collect();
@@ -1053,9 +1020,7 @@ impl DiskLocation {
// propagate the error to the caller.
let created = !self.ec_volumes.contains_key(&vid);
if created {
let ec_vol =
EcVolume::new_with_vif_dirs(&dir, idx_dir, &self.store_dirs, collection, vid)
.map_err(VolumeError::Io)?;
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid).map_err(VolumeError::Io)?;
self.ec_volumes.insert(vid, ec_vol);
}
let ec_vol = self.ec_volumes.get_mut(&vid).expect("just inserted above");
@@ -1476,7 +1441,6 @@ fn rm_if_present(path: String) -> io::Result<()> {
fn ec_data_shards_from_vif(
directory: &str,
idx_directory: &str,
store_dirs: &[String],
collection: &str,
vid: VolumeId,
) -> usize {
@@ -1495,23 +1459,6 @@ fn ec_data_shards_from_vif(
break;
}
}
match crate::storage::erasure_coding::ec_volume::sibling_ec_config_vif(
directory,
idx_directory,
store_dirs,
collection,
vid,
) {
Err(()) => return 0,
Ok(Some(vi)) => {
if let Some(ds) = vi.ec_shard_config.map(|c| c.data_shards as usize)
&& ds > 0
{
return ds;
}
}
Ok(None) => {}
}
DATA_SHARDS_COUNT
}
@@ -1595,7 +1542,7 @@ pub(crate) fn vif_references_remote_file(vif_path: &str) -> bool {
/// True when a `.vif` records an EC shard config, i.e. the volume was
/// erasure-coded. Such a volume keeps no local `.dat`, so an empty `.dat`
/// alongside it is a leftover stub safe to remove.
pub(crate) fn vif_is_ec_volume(vif_path: &str) -> bool {
fn vif_is_ec_volume(vif_path: &str) -> bool {
fs::read_to_string(vif_path)
.ok()
.and_then(|s| serde_json::from_str::<VifVolumeInfo>(&s).ok())
@@ -272,96 +272,6 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume
data_vif
}
/// [`locate_vif_path`], except that a data-dir `.vif` recording no EC config
/// yields to the idx dir's when that one records it: a source volume's own
/// `.vif` stays on a disk that later receives shards of the same volume id,
/// while the encode's `.vif` lands with the index on a sibling.
fn resolve_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
let path = locate_vif_path(dir, dir_idx, collection, volume_id);
let data_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir, collection, volume_id),
);
if dir_idx == dir || path != data_vif {
return path;
}
let idx_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
);
prefer_ec_config_vif(path, idx_vif)
}
fn read_vif_file(path: &str) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
match std::fs::read_to_string(path) {
Ok(content) if content.trim().is_empty() => Ok(None),
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|_| ()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
Err(_) => Err(()),
}
}
/// Returns `alt` when `local` records no EC config but `alt` does.
fn prefer_ec_config_vif(local: String, alt: String) -> String {
match read_vif_file(&local) {
Ok(Some(info)) if info.ec_shard_config.is_some() => return local,
Err(()) => return local,
_ => {}
}
match read_vif_file(&alt) {
Ok(Some(info)) if info.ec_shard_config.is_some() => alt,
_ => local,
}
}
/// The `.vif` a disk mounting against a shared `-dir.idx` takes from a sibling
/// data dir in `vif_dirs`: its own records no EC config, the index is not its
/// own, and every sibling `.vif` recording an EC config names the same run.
/// `Err` when a sibling `.vif` cannot be read or the siblings name different runs.
pub(crate) fn sibling_ec_config_vif(
dir: &str,
dir_idx: &str,
vif_dirs: &[String],
collection: &str,
volume_id: VolumeId,
) -> Result<Option<crate::storage::volume::VifVolumeInfo>, ()> {
let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let data_vif = format!("{}.vif", data_base);
if dir_idx == dir
|| vif_dirs.iter().any(|d| d == dir_idx)
|| matches!(ecx_file_size(&format!("{}.ecx", data_base)), Some(n) if n > 0)
|| resolve_vif_path(dir, dir_idx, collection, volume_id) != data_vif
{
return Ok(None);
}
match read_vif_file(&data_vif) {
Ok(Some(info)) if info.ec_shard_config.is_some() => return Ok(None),
Err(()) => return Ok(None),
_ => {}
}
let mut found: Option<crate::storage::volume::VifVolumeInfo> = None;
for other in vif_dirs.iter().filter(|d| d.as_str() != dir) {
let path = format!(
"{}.vif",
crate::storage::volume::volume_file_name(other, collection, volume_id)
);
let info = match read_vif_file(&path) {
Ok(Some(info)) if info.ec_shard_config.is_some() => info,
Ok(_) => continue,
Err(()) => return Err(()),
};
let run = |v: &crate::storage::volume::VifVolumeInfo| {
v.ec_shard_config.as_ref().map(|c| c.encode_ts_ns)
};
match &found {
None => found = Some(info),
Some(f) if run(f) != run(&info) => return Err(()),
Some(_) => {}
}
}
Ok(found)
}
/// Load the volume's `.vif` (data dir first, then idx dir, then the active
/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
/// everywhere is legal — legacy volumes predate the sidecar — and returns
@@ -376,7 +286,7 @@ pub fn load_vif_info(
collection: &str,
volume_id: VolumeId,
) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
let vif_path = resolve_vif_path(dir, dir_idx, collection, volume_id);
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
match std::fs::read_to_string(&vif_path) {
Ok(content) if content.trim().is_empty() => Ok(None),
Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
@@ -426,16 +336,15 @@ pub fn load_vif_info_across_dirs(
// pointed callers at the wrong disk whenever the vif lived with the
// index — and a caller resolving the rest of the volume's metadata
// against that answer would look in a directory holding none of it.
let idx_vif = format!(
let data_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id)
crate::storage::volume::volume_file_name(dir, collection, volume_id)
);
let found_in =
if dir_idx != dir && resolve_vif_path(dir, dir_idx, collection, volume_id) == idx_vif {
dir_idx.to_string()
} else {
dir.to_string()
};
let found_in = if std::path::Path::new(&data_vif).exists() {
dir.to_string()
} else {
dir_idx.to_string()
};
return Ok(Some((vif, found_in)));
}
for other in other_dirs {
@@ -660,25 +569,7 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL, &[])
}
/// [`Self::new`] for a disk whose server's data directories are `vif_dirs`.
pub fn new_with_vif_dirs(
dir: &str,
dir_idx: &str,
vif_dirs: &[String],
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
Self::open_with(
dir,
dir_idx,
collection,
volume_id,
EcjFsOps::REAL,
vif_dirs,
)
Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL)
}
/// [`Self::new`] with the journal-compaction filesystem steps supplied, so
@@ -689,15 +580,10 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
ecj_ops: EcjFsOps,
vif_dirs: &[String],
) -> io::Result<Self> {
// One load of the volume's `.vif`, used for both the shard config and
// the version / dat-size fields below.
let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
let vif = sibling_ec_config_vif(dir, dir_idx, vif_dirs, collection, volume_id)
.ok()
.flatten()
.or(vif);
// Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
// the INDEX, and it is the only layout record a volume whose vif is
// absent or config-free still has.
@@ -3694,7 +3580,7 @@ mod tests {
);
let before = std::fs::read(&ecj_path).unwrap();
(
EcVolume::open_with(dir, dir, "", vid, ops, &[]),
EcVolume::open_with(dir, dir, "", vid, ops),
ecj_path,
before,
)
@@ -3915,7 +3801,7 @@ mod tests {
read_at: rewrite_after_last_read,
..EcjFsOps::REAL
};
let vol = EcVolume::open_with(dir, dir, "", vid, ops, &[]).unwrap();
let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap();
assert!(
IN_PLACE_REWRITE.with(|r| r.borrow().is_none()),
"the rewrite never ran"
@@ -4222,143 +4108,6 @@ mod tests {
assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
}
/// Shards placed on the disk that still holds the source volume's `.vif`
/// mount against an index on a sibling disk; the encode identity and
/// layout come from the sibling's `.vif`, not the source's config-free one.
#[test]
fn test_ec_volume_prefers_idx_vif_over_config_free_data_vif() {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
let data_base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", data_base),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
dat_file_size: 4096,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 12,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_123_456_789,
block_size: 3 * 1024 * 1024,
}),
..Default::default()
};
let idx_base = crate::storage::volume::volume_file_name(dir_idx, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", idx_base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, 1_700_000_000_123_456_789);
assert_eq!((vol.data_shards, vol.parity_shards), (12, 4));
assert_eq!(vol.block_size, 3 * 1024 * 1024);
assert_eq!(vol.dat_file_size, 4096);
let (_, found_in) = load_vif_info_across_dirs(dir, dir_idx, &[], "pics", VolumeId(1))
.unwrap()
.unwrap();
assert_eq!(found_in, dir_idx);
}
/// A data-dir `.vif` that records an EC config stays authoritative.
#[test]
fn test_ec_volume_keeps_data_vif_with_ec_config() {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
write_ecx_file(dir_idx, "pics", VolumeId(1), &[]);
for (d, ts) in [(dir, 11), (dir_idx, 22)] {
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: ts,
..Default::default()
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(d, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
}
let vol = EcVolume::new(dir, dir_idx, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, 11);
}
/// With a shared `-dir.idx` the encode's `.vif` sits in the data dir of the
/// disk that received the index, so a disk holding a source volume's
/// config-free `.vif` takes it from that sibling, but only when the
/// siblings agree on one run.
#[test]
fn test_ec_volume_takes_sibling_vif_with_shared_index() {
const RUN_A: i64 = 1_700_000_000_000_000_011;
const RUN_B: i64 = 1_700_000_000_000_000_022;
let cases: [(&str, &[i64], bool, i64); 4] = [
("one sibling", &[RUN_A], false, RUN_A),
("siblings agree", &[RUN_A, RUN_A], false, RUN_A),
("siblings disagree", &[RUN_A, RUN_B], false, 0),
("local index", &[RUN_A], true, 0),
];
for (name, sibling_ts, local_index, want) in cases {
let data_tmp = TempDir::new().unwrap();
let idx_tmp = TempDir::new().unwrap();
let dir = data_tmp.path().to_str().unwrap();
let dir_idx = idx_tmp.path().to_str().unwrap();
let entry = [(NeedleId(1), Offset::from_actual_offset(8), Size(10))];
write_ecx_file(dir_idx, "c", VolumeId(1), &entry);
if local_index {
write_ecx_file(dir, "c", VolumeId(1), &entry);
}
let data_base = crate::storage::volume::volume_file_name(dir, "c", VolumeId(1));
std::fs::write(
format!("{}.vif", data_base),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
let mut dirs = vec![dir.to_string()];
let mut siblings = Vec::new();
for &ts in sibling_ts {
let sibling = TempDir::new().unwrap();
let sibling_dir = sibling.path().to_str().unwrap().to_string();
let vif = crate::storage::volume::VifVolumeInfo {
version: 3,
dat_file_size: 4096,
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: ts,
..Default::default()
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(&sibling_dir, "c", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
dirs.push(sibling_dir);
siblings.push(sibling);
}
let vol = EcVolume::new_with_vif_dirs(dir, dir_idx, &dirs, "c", VolumeId(1)).unwrap();
assert_eq!(vol.encode_ts_ns, want, "{}", name);
}
}
#[test]
fn test_ec_volume_wide_ratio_vif_config() {
let tmp = TempDir::new().unwrap();
+49 -370
View File
@@ -13,6 +13,7 @@ use std::io::{self, Read, Seek, Write};
use std::path::Path;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
#[cfg(feature = "redb-experimental-cursor")]
use std::ops::Bound;
mod compact_map;
@@ -484,38 +485,6 @@ const NEEDLE_TABLE: TableDefinition<u64, &[u8]> = TableDefinition::new("needles"
const META_TABLE: TableDefinition<&str, u64> = TableDefinition::new("meta");
const META_IDX_SIZE: &str = "idx_size";
/// redb 4.6 starts a new file at 72 KiB: 64 KiB of pages plus the header
/// and the region tracker.
const REDB_SMALL_FILE_BYTES: u64 = 72 * 1024;
/// redb 4.2 started at 1 MiB of pages plus that same 8 KiB.
const REDB_OLD_FLOOR_BYTES: u64 = 1024 * 1024 + 8 * 1024;
/// Meta value `1` means a compact of a file in the old-floor band already finished.
const META_FLOOR_COMPACT: &str = "floor_compact";
/// True when a reused `.rdb` is still inside the old 4.2 starting size.
/// A file one byte past it has grown.
fn should_compact_old_floor(len: u64) -> bool {
len > REDB_SMALL_FILE_BYTES && len <= REDB_OLD_FLOOR_BYTES
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CompactFollowUp {
/// Drop the `Database` and open the same path once. Do not compact again.
ReopenSkipCompact,
/// The handle can still begin a write. Do not record `floor_compact`.
KeepHandle,
}
fn compact_follow_up(err: &redb::CompactionError) -> CompactFollowUp {
match err {
redb::CompactionError::Storage(_) => CompactFollowUp::ReopenSkipCompact,
redb::CompactionError::TransactionInProgress
| redb::CompactionError::PersistentSavepointExists
| redb::CompactionError::EphemeralSavepointExists => CompactFollowUp::KeepHandle,
_ => CompactFollowUp::KeepHandle,
}
}
/// Writes between two durable redb checkpoints. Every non-durable commit
/// leaves an entry in redb's transaction tracker and pins the pages that
/// were on disk at the last durable commit; only a durable commit clears
@@ -679,6 +648,9 @@ impl RedbNeedleMap {
let meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
// ReadableTable::get guard borrows `meta`; edition 2024 drops the tail
// expression's temporaries before `meta`, so no extra binding is needed.
match meta.get(META_IDX_SIZE) {
Ok(Some(guard)) => Ok(Some(guard.value())),
Ok(None) => Ok(None),
@@ -686,118 +658,6 @@ impl RedbNeedleMap {
}
}
fn rdb_file_len(&self) -> io::Result<u64> {
Ok(std::fs::metadata(&self.rdb_path)?.len())
}
fn read_floor_compact(&self) -> io::Result<Option<u64>> {
let txn = self
.db_or_err()?
.begin_read()
.map_err(|e| io::Error::other(format!("redb begin_read: {}", e)))?;
let meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
match meta.get(META_FLOOR_COMPACT) {
Ok(Some(guard)) => Ok(Some(guard.value())),
Ok(None) => Ok(None),
Err(e) => Err(io::Error::other(format!("redb get meta: {}", e))),
}
}
fn save_floor_compact(&self) -> io::Result<()> {
let txn = Self::begin_write_no_fsync(self.db_or_err()?)?;
{
let mut meta = txn
.open_table(META_TABLE)
.map_err(|e| io::Error::other(format!("redb open meta: {}", e)))?;
meta.insert(META_FLOOR_COMPACT, 1u64)
.map_err(|e| io::Error::other(format!("redb insert meta: {}", e)))?;
}
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit meta: {}", e)))?;
Ok(())
}
/// Drop the exclusive lock and open the same file. Does not compact.
fn reopen_database(&mut self) -> io::Result<()> {
self.db.take();
let db = Database::builder()
.set_cache_size(self.cache_bytes)
.open(&self.rdb_path)
.map_err(|e| io::Error::other(format!("redb reopen after compact: {}", e)))?;
self.db = Some(db);
Ok(())
}
fn record_floor_compact_if_still_in_band(&self) -> io::Result<()> {
let len = match self.rdb_file_len() {
Ok(len) => len,
Err(e) => {
tracing::warn!("redb floor compact flag skipped, metadata failed: {}", e);
return Ok(());
}
};
if should_compact_old_floor(len) {
self.save_floor_compact()?;
}
Ok(())
}
/// Compact a reused file that is still the 4.2 starting allocation.
///
/// Called only from `try_reuse_rdb`, after the replay transaction has
/// committed. `new` and `full_rebuild` already create a 72 KiB file.
///
/// An empty durable commit publishes a `Durability::None` tail before
/// `compact`. A storage error inside `compact` would otherwise drop that
/// tail when the handle is reopened. `checkpoint` is not used: during
/// reuse `idx_file` is `None` and `idx_file_offset` is 0.
fn shrink_old_floor(&mut self) -> io::Result<()> {
let len = match self.rdb_file_len() {
Ok(len) => len,
Err(e) => {
tracing::warn!("redb floor compact skipped, metadata failed: {}", e);
return Ok(());
}
};
// The flag is irrelevant outside the band. A meta read error there
// must not make `load_from_idx` rebuild a larger index.
if !should_compact_old_floor(len) {
return Ok(());
}
if self.read_floor_compact()? == Some(1) {
return Ok(());
}
// Failure returns Err so `load_from_idx` full-rebuilds. Do not reopen
// over a tail that this commit did not publish.
let mut txn = self
.db_or_err()?
.begin_write()
.map_err(|e| io::Error::other(format!("redb begin_write: {}", e)))?;
txn.set_quick_repair(true);
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
let compact_result = self
.db
.as_mut()
.ok_or_else(|| io::Error::other("redb database is closed"))?
.compact();
match compact_result {
Ok(_) => self.record_floor_compact_if_still_in_band(),
Err(e) => match compact_follow_up(&e) {
CompactFollowUp::ReopenSkipCompact => {
tracing::warn!("redb floor compact storage error, reopening: {}", e);
self.reopen_database()
}
CompactFollowUp::KeepHandle => {
tracing::warn!("redb floor compact skipped: {}", e);
Ok(())
}
},
}
}
/// Test-only read of META `idx_size` through the live handle. See
/// [`test_support::live_meta_idx_size`] for why durability tests use
/// this instead of copying the open `.rdb`.
@@ -806,11 +666,6 @@ impl RedbNeedleMap {
self.read_idx_size_meta().unwrap()
}
#[cfg(test)]
fn live_floor_compact(&self) -> Option<u64> {
self.read_floor_compact().unwrap()
}
/// Load from an .idx file, reusing an existing .rdb if it is consistent.
///
/// Strategy:
@@ -919,7 +774,6 @@ impl RedbNeedleMap {
nm.save_idx_size_meta(idx_size)?;
}
nm.shrink_old_floor()?;
Ok(nm)
}
@@ -942,8 +796,8 @@ impl RedbNeedleMap {
/// `Database::create` opens an existing file (`truncate(false)`). Unlink
/// is best-effort: if it fails (sticky bit, foreign owner) but the path is
/// still writable, clear the leftover needles table before inserting.
/// Live keys come from a `BTreeMap` so `insert_before` walks them in
/// needle-id order and packs leaves, without a second Vec + sort scratch.
/// Live keys come from a `BTreeMap` so inserts are in needle-id order and
/// redb 4.2.0 packs leaves, without a second Vec + sort scratch.
fn full_rebuild<R: Read + Seek>(
db_path: &str,
reader: &mut R,
@@ -995,26 +849,43 @@ impl RedbNeedleMap {
.map_err(|e| io::Error::other(format!("redb retain: {}", e)))?;
}
let mut cursor = table
.upper_bound_mut(Bound::<u64>::Unbounded)
.map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb upper_bound_mut: {}", e))
#[cfg(not(feature = "redb-experimental-cursor"))]
{
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
table
.insert(key_u64, packed.as_slice())
.map_err(|e| io::Error::other(format!("redb insert: {}", e)))?;
}
}
#[cfg(feature = "redb-experimental-cursor")]
{
let mut cursor =
table
.upper_bound_mut(Bound::<u64>::Unbounded)
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb upper_bound_mut: {}", e),
)
})?;
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
cursor
.insert_before(key_u64, packed.as_slice())
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb insert_before: {}", e),
)
})?;
}
cursor.close().map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
})?;
for (key, nv) in &entries {
let key_u64: u64 = (*key).into();
let packed = pack_needle_value(nv);
cursor
.insert_before(key_u64, packed.as_slice())
.map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("redb insert_before: {}", e),
)
})?;
}
cursor.close().map_err(|e| {
io::Error::new(io::ErrorKind::Other, format!("redb cursor close: {}", e))
})?;
}
txn.commit()
.map_err(|e| io::Error::other(format!("redb commit: {}", e)))?;
@@ -1177,6 +1048,9 @@ impl RedbNeedleMap {
let table = txn
.open_table(NEEDLE_TABLE)
.map_err(|e| io::Error::other(format!("redb open_table: {}", e)))?;
// experimental-api-5 drops inherent ReadOnlyTable::get ('static guard).
// ReadableTable::get guard borrows `table`; edition 2024 drops the tail
// expression's temporaries before `table`, so no extra binding is needed.
match table.get(key_u64) {
Ok(Some(guard)) => Ok(packed_to_needle_value(guard.value())),
Ok(None) => Ok(None),
@@ -1307,7 +1181,7 @@ impl RedbNeedleMap {
/// Reopen the database from `.idx` after a non-`TransactionPoisoned`
/// commit error.
///
/// redb can make a `Durability::None` commit visible *before*
/// redb 4.2.0 can make a `Durability::None` commit visible *before*
/// returning `Err(CommitError::Storage(..))`. In that state the database
/// refuses further write transactions, so the map cannot be used as-is.
/// The `.idx` row that triggered the commit is **not** truncated — the
@@ -1729,10 +1603,9 @@ pub(crate) mod test_support {
/// every `put`/`delete` also commits non-durably, so before the first
/// checkpoint this is the load-time value (`Some(0)` for a fresh map) —
/// NOT the crash-durable `None` a copy of the open `.rdb` would show.
/// A live read is the only portable observation: redb's default
/// `ExclusiveWriter` takes an exclusive whole-file lock, which is
/// advisory on Unix but mandatory on Windows, so copying the open
/// `.rdb` fails there with OS error 33.
/// A live read is the only portable observation: redb 4.2.0 takes an
/// exclusive whole-file lock, which is advisory on Unix but mandatory
/// on Windows, so copying the open `.rdb` fails there with OS error 33.
///
/// It still pins the property under test: the only *durable* META
/// writer is `checkpoint`, so any value other than the load-time one
@@ -1890,41 +1763,6 @@ mod tests {
NeedleMapKind::Redb.redb_cache_bytes()
}
#[test]
fn test_redb_new_file_is_at_most_72kib() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("fresh.rdb");
let nm = RedbNeedleMap::new(db_path.to_str().unwrap(), redb_test_cache()).unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
assert!(
len <= REDB_SMALL_FILE_BYTES,
"fresh redb file is {len} bytes, above the 72 KiB start"
);
drop(nm);
}
#[test]
fn test_should_compact_old_floor_bounds() {
let small = REDB_SMALL_FILE_BYTES;
let floor = REDB_OLD_FLOOR_BYTES;
assert!(!should_compact_old_floor(small));
assert!(should_compact_old_floor(small + 1));
assert!(should_compact_old_floor(floor));
assert!(!should_compact_old_floor(floor + 1));
assert!(!should_compact_old_floor(0));
}
fn reload_redb(db_path: &std::path::Path, idx_path: &std::path::Path) -> RedbNeedleMap {
let mut idx = std::fs::File::open(idx_path).unwrap();
RedbNeedleMap::load_from_idx(
db_path.to_str().unwrap(),
&mut idx,
Version::current(),
redb_test_cache(),
)
.unwrap()
}
/// Open a redb map on an empty .idx with an append writer attached, the
/// way `Volume::load_index_redb` opens a writable volume.
fn open_writable_redb(
@@ -2057,6 +1895,7 @@ mod tests {
);
}
#[cfg(feature = "redb-experimental-cursor")]
#[test]
fn test_redb_full_rebuild_insert_before_reads_back_thousands_of_shuffled_keys() {
// Enough keys to split leaves. insert_before vs table.insert only
@@ -2635,164 +2474,4 @@ mod tests {
);
}
}
#[test]
fn test_redb_fresh_reuse_does_not_record_floor_compact() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let len = std::fs::metadata(&db_path).unwrap().len();
assert!(len <= REDB_SMALL_FILE_BYTES, "fresh file is {len} bytes");
nm.close();
drop(nm);
let loaded = reload_redb(&db_path, &idx_path);
assert_eq!(loaded.live_floor_compact(), None);
assert!(loaded.get(NeedleId(1)).unwrap().is_none());
}
#[test]
fn test_redb_in_band_file_is_compacted_once() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let mut n = 0u64;
loop {
n += 1;
nm.put(
NeedleId(n),
Offset::from_actual_offset((n as i64).saturating_mul(8)),
Size(8),
)
.unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
if should_compact_old_floor(len) {
break;
}
assert!(
n < 2_000_000,
"file grew to {len} bytes without entering the old-floor band"
);
}
nm.checkpoint(true).unwrap();
nm.close();
drop(nm);
let mut loaded = reload_redb(&db_path, &idx_path);
let len = std::fs::metadata(&db_path).unwrap().len();
let flag = loaded.live_floor_compact();
if should_compact_old_floor(len) {
assert_eq!(flag, Some(1));
} else {
assert!(len <= REDB_SMALL_FILE_BYTES, "compacted file is {len} bytes");
assert_eq!(flag, None);
}
let first = loaded.get(NeedleId(1)).unwrap();
assert!(first.is_some());
let last = loaded.get(NeedleId(n)).unwrap();
assert!(last.is_some());
// close checkpoints the non-durable floor_compact flag, then drop
// releases the exclusive lock before the second open.
loaded.close();
drop(loaded);
let again = reload_redb(&db_path, &idx_path);
assert_eq!(std::fs::metadata(&db_path).unwrap().len(), len);
assert_eq!(again.live_floor_compact(), flag);
}
#[test]
fn test_redb_sparse_old_floor_shrinks_on_reuse() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, db_path, idx_path) = open_writable_redb(dir.path());
let mut n = 0u64;
loop {
n += 1;
nm.put(
NeedleId(n),
Offset::from_actual_offset((n as i64).saturating_mul(8)),
Size(8),
)
.unwrap();
let len = std::fs::metadata(&db_path).unwrap().len();
if should_compact_old_floor(len) {
break;
}
assert!(n < 2_000_000, "file is {len} bytes and never entered the band");
}
assert!(n > 32, "band was reached with only {n} keys");
nm.checkpoint(true).unwrap();
nm.close();
drop(nm);
let keep = 32u64;
let idx = std::fs::OpenOptions::new().write(true).open(&idx_path).unwrap();
idx.set_len(keep * NEEDLE_MAP_ENTRY_SIZE as u64).unwrap();
drop(idx);
let db = redb::Database::builder()
.set_cache_size(redb_test_cache())
.open(db_path.to_str().unwrap())
.unwrap();
let txn = db.begin_write().unwrap();
{
let mut table = txn.open_table(NEEDLE_TABLE).unwrap();
for key in (keep + 1)..=n {
table.remove(key).unwrap();
}
}
{
let mut meta = txn.open_table(META_TABLE).unwrap();
meta.insert(META_IDX_SIZE, keep * NEEDLE_MAP_ENTRY_SIZE as u64)
.unwrap();
}
txn.commit().unwrap();
drop(db);
let before = std::fs::metadata(&db_path).unwrap().len();
let loaded = reload_redb(&db_path, &idx_path);
let after = std::fs::metadata(&db_path).unwrap().len();
assert!(after < before, "reuse left the file at {after} bytes, was {before}");
assert!(
after <= REDB_SMALL_FILE_BYTES,
"32 live keys still occupy {after} bytes"
);
assert!(loaded.get(NeedleId(1)).unwrap().is_some());
assert!(loaded.get(NeedleId(keep)).unwrap().is_some());
assert!(loaded.get(NeedleId(keep + 1)).unwrap().is_none());
}
#[test]
fn test_compact_follow_up_reopens_only_on_storage_error() {
use redb::{CompactionError, StorageError};
let storage = CompactionError::Storage(StorageError::Io(std::io::Error::other("sync")));
assert_eq!(
compact_follow_up(&storage),
CompactFollowUp::ReopenSkipCompact
);
assert_eq!(
compact_follow_up(&CompactionError::TransactionInProgress),
CompactFollowUp::KeepHandle
);
assert_eq!(
compact_follow_up(&CompactionError::PersistentSavepointExists),
CompactFollowUp::KeepHandle
);
assert_eq!(
compact_follow_up(&CompactionError::EphemeralSavepointExists),
CompactFollowUp::KeepHandle
);
}
#[test]
fn test_redb_reopen_database_reads_committed_key() {
let dir = tempfile::tempdir().unwrap();
let (mut nm, _db_path, _idx_path) = open_writable_redb(dir.path());
nm.put(NeedleId(7), Offset::from_actual_offset(64), Size(9))
.unwrap();
nm.reopen_database().unwrap();
let nv = nm.get(NeedleId(7)).unwrap().unwrap();
assert_eq!(nv.size, Size(9));
nm.put(NeedleId(8), Offset::from_actual_offset(128), Size(10))
.unwrap();
let nv = nm.get(NeedleId(8)).unwrap().unwrap();
assert_eq!(nv.size, Size(10));
}
}
+14 -118
View File
@@ -122,9 +122,6 @@ pub struct Store {
/// The copy-claim set installed on every location by
/// set_volume_load_guard, kept here so a later add_location gets it too.
load_guard: Option<Arc<Mutex<HashSet<VolumeId>>>>,
/// The data directory of every disk this server loads, set before the
/// first `add_location` so a disk's EC mounts can see its siblings.
pub store_dirs: Vec<String>,
}
impl Store {
@@ -151,7 +148,6 @@ impl Store {
disk_stats_of: super::disk_location::get_disk_stats,
load_guard: None,
preallocate_supported: PREALLOCATE_SUPPORTED,
store_dirs: Vec::new(),
}
}
@@ -174,7 +170,6 @@ impl Store {
tags,
)?;
loc.load_guard = self.load_guard.clone();
loc.store_dirs = self.store_dirs.clone();
loc.load_existing_volumes(self.needle_map_kind)?;
// Check for duplicate volume IDs across existing locations
@@ -292,20 +287,12 @@ impl Store {
// ---- Volume lifecycle ----
/// Find the location with the fewest writable volumes of `collection`, so
/// they do not share one disk's write bandwidth, then the most free slots
/// (load-balance) of the given disk type. Matches Go's FindFreeLocation:
/// accounts for EC shards when computing free slots.
fn find_free_location(
&self,
disk_type: &DiskType,
collection: &str,
preallocate: u64,
) -> Option<usize> {
/// Find the location with fewest volumes (load-balance) of the given disk type.
/// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots.
fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option<usize> {
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed);
let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free)
let mut best: Option<(usize, i64)> = None; // (index, effective_free)
for (i, loc) in self.locations.iter().enumerate() {
if &loc.disk_type != disk_type {
continue;
@@ -335,14 +322,11 @@ impl Store {
) {
continue;
}
let writable = loc.writable_collection_volumes_len(collection, volume_size_limit);
if best.is_none_or(|(_, min_writable, max_free)| {
writable < min_writable || (writable == min_writable && effective_free > max_free)
}) {
best = Some((i, writable, effective_free));
if best.is_none() || effective_free > best.unwrap().1 {
best = Some((i, effective_free));
}
}
best.map(|(i, _, _)| i)
best.map(|(i, _)| i)
}
/// Find a free location matching a predicate.
@@ -612,7 +596,7 @@ impl Store {
return Err(VolumeError::AlreadyExists);
}
let loc_idx = self
.find_free_location(&disk_type, spec.collection, spec.preallocate)
.find_free_location(&disk_type, spec.preallocate)
.ok_or_else(|| {
VolumeError::Io(io::Error::other(format!(
"no free location for disk type {:?}",
@@ -1053,18 +1037,13 @@ impl Store {
}
/// Delete a needle from a volume.
pub fn delete_volume_needle(
&self,
vid: VolumeId,
n: &mut Needle,
fsync: bool,
) -> Result<Size, VolumeError> {
pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result<Size, VolumeError> {
// Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
if vol.is_no_write_or_delete() {
return Err(VolumeError::ReadOnly(vid));
}
vol.delete_needle(n, fsync)
vol.delete_needle(n)
}
// ---- Collection operations ----
@@ -2939,9 +2918,7 @@ mod tests {
cookie: Cookie(0xaa),
..Needle::default()
};
let deleted = store
.delete_volume_needle(VolumeId(1), &mut del_n, false)
.unwrap();
let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap();
assert!(deleted.0 > 0);
}
@@ -3050,87 +3027,6 @@ mod tests {
assert_eq!(store.locations[1].volumes_len(), 1);
}
fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) {
let spec = VolumeSpec {
collection,
..VolumeSpec::default()
};
store
.add_volume(VolumeId(vid), DiskType::HardDrive, &spec)
.unwrap();
}
fn location_of(store: &Store, vid: u32) -> Option<usize> {
store
.locations
.iter()
.position(|loc| loc.find_volume(VolumeId(vid)).is_some())
}
// Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks.
#[test]
fn test_add_volume_spreads_collection_over_disks() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
add_spread_test_volume(&mut store, 1, "other");
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
let first = location_of(&store, 3);
// Leave the disk with the "spread" volume holding fewer volumes overall.
let other_on_first = if first == Some(1) { 2 } else { 1 };
store
.delete_volume(VolumeId(other_on_first), false, false, false)
.unwrap();
add_spread_test_volume(&mut store, 4, "spread");
assert_ne!(location_of(&store, 4), first);
}
// Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection.
#[test]
fn test_add_volume_ignores_full_volumes_of_collection() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
// Every .dat holds at least its superblock, so all volumes are full.
store.volume_size_limit.store(1, Ordering::Relaxed);
add_spread_test_volume(&mut store, 1, "spread");
let full_disk = location_of(&store, 1);
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
assert_eq!(location_of(&store, 3), full_disk);
}
// Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection.
#[test]
fn test_add_volume_ignores_read_only_volumes_of_collection() {
let tmp1 = TempDir::new().unwrap();
let tmp2 = TempDir::new().unwrap();
let mut store =
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
add_spread_test_volume(&mut store, 1, "spread");
let writable_disk = location_of(&store, 1);
add_spread_test_volume(&mut store, 2, "other");
add_spread_test_volume(&mut store, 3, "spread");
assert_ne!(location_of(&store, 3), writable_disk);
store
.find_volume(VolumeId(3))
.unwrap()
.1
.set_read_only()
.unwrap();
// The other disk holds more volumes, but its "spread" volume is read-only.
add_spread_test_volume(&mut store, 4, "spread");
assert_ne!(location_of(&store, 4), writable_disk);
}
#[test]
fn test_store_delete_collection() {
let tmp = TempDir::new().unwrap();
@@ -3499,7 +3395,7 @@ mod tests {
cookie: Cookie(2),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap();
@@ -3823,7 +3719,7 @@ mod tests {
cookie: Cookie(2),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
let revision = {
let (_, v) = store.find_volume(vid).unwrap();
v.super_block().compaction_revision
@@ -3848,7 +3744,7 @@ mod tests {
cookie: Cookie(4),
..Needle::default()
};
store.delete_volume_needle(vid, &mut del, false).unwrap();
store.delete_volume_needle(vid, &mut del).unwrap();
drop(release_tx);
copy.join().unwrap().unwrap();
+12 -270
View File
@@ -8,9 +8,7 @@ use std::path::Path;
use tracing::{info, warn};
use crate::storage::disk_location::{
DiskLocation, parse_collection_volume_id_pub, vif_is_ec_volume,
};
use crate::storage::disk_location::{DiskLocation, parse_collection_volume_id_pub};
use crate::storage::store::Store;
use crate::storage::types::VolumeId;
@@ -36,14 +34,10 @@ impl Store {
/// `reconcile_ec_shards_across_disks` so the orphan pass can
/// prefer the local idx_directory.
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
self.mirror_ec_metadata_to_shard_disks_only(None)
}
pub(crate) fn mirror_ec_metadata_to_shard_disks_only(&mut self, only: Option<VolumeId>) {
if self.locations.len() < 2 {
return;
}
let owners = self.index_ecx_owners_for_mirror(only);
let owners = self.index_ecx_owners_for_mirror();
if owners.is_empty() {
return;
}
@@ -58,7 +52,7 @@ impl Store {
}
let mut mirrors: Vec<Mirror> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let orphans = collect_shard_disk_volumes(loc, only);
let orphans = collect_shard_disk_volumes(loc);
for (key, _shards) in orphans {
let Some(owner) = owners.get(&key) else {
continue;
@@ -117,7 +111,7 @@ impl Store {
// Records both idx_dir (where .ecx was found) and data_dir, so
// the mirror can resolve .vif from data_dir even when .ecx lives
// in idx_directory.
fn index_ecx_owners_for_mirror(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwner> {
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let mut seen: Vec<&str> = Vec::with_capacity(2);
@@ -137,18 +131,15 @@ impl Store {
let Some(base) = name.strip_suffix(".ecx") else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
// credible owner — skip it so the scan keeps looking for a
// real index on a sibling disk (Go's indexEcxOwners).
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
owners
.entry(EcKey { collection, vid })
.or_insert_with(|| EcxOwner {
@@ -216,33 +207,6 @@ fn mirror_sidecars_for_volume(
collection: &str,
vid: VolumeId,
) -> io::Result<usize> {
// A local .vif with no EC config (a source volume's own) would shadow the
// owner's once the index is copied next to it; keep mounting across disks.
let dst_vif = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ".vif");
if path_is_regular_file(&dst_vif)
&& !vif_is_ec_volume(&dst_vif)
&& (vif_is_ec_volume(&sidecar_dest_path(
src_data_dir,
src_idx_dir,
collection,
vid,
".vif",
)) || vif_is_ec_volume(&sidecar_dest_path(
src_idx_dir,
src_data_dir,
collection,
vid,
".vif",
)))
{
info!(
volume_id = vid.0,
collection = %collection,
vif = %dst_vif,
"local .vif records no EC config; not mirroring EC sidecars",
);
return Ok(0);
}
let mut copied = 0usize;
for ext in EC_MIRRORED_SIDECARS {
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
@@ -304,10 +268,7 @@ fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
Ok(())
}
fn collect_shard_disk_volumes(
loc: &DiskLocation,
only: Option<VolumeId>,
) -> HashMap<EcKey, Vec<String>> {
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
let Ok(read) = fs::read_dir(&loc.directory) else {
return out;
@@ -324,16 +285,13 @@ fn collect_shard_disk_volumes(
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
match ent.metadata() {
Ok(meta) if meta.len() > 0 => {}
_ => continue,
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
out.entry(EcKey { collection, vid }).or_default().push(name);
}
out
@@ -433,222 +391,6 @@ mod tests {
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
}
/// Shards on a disk that still holds the source volume's config-free
/// `.vif` must mount with the owner's encode identity: mirroring the index
/// next to that `.vif` would make the disk mount through it.
#[test]
fn mirror_skips_disk_with_config_free_vif() {
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
fs::create_dir_all(&dir0).unwrap();
fs::create_dir_all(&dir1).unwrap();
let collection = "c";
let vid = 4122u32;
plant_shard(&dir0, collection, vid, 2);
plant_shard(&dir1, collection, vid, 0);
fs::write(
dir0.join(format!("{}_{}.vif", collection, vid)),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&dir1, collection, vid, &[0xA1u8; 20]);
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_000_000_042,
..Default::default()
}),
..Default::default()
};
fs::write(
dir1.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
add_loc(&mut store, &dir0);
add_loc(&mut store, &dir1);
assert!(
!dir0.join(format!("{}_{}.ecx", collection, vid)).exists(),
".ecx mirrored next to the config-free .vif"
);
let ec_vol = store.locations[0]
.find_ec_volume(VolumeId(vid))
.expect("dir0 ec volume mounted");
assert_eq!(ec_vol.encode_ts_ns, 1_700_000_000_000_000_042);
}
/// With a shared `-dir.idx` every disk sees the `.ecx`, and only the disk
/// that received it holds the encode's `.vif`. A disk loaded first that
/// still holds the source volume's config-free `.vif` must take that
/// identity from its sibling.
#[test]
fn shared_index_disk_takes_sibling_vif_on_startup() {
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
let idx = tmp.path().join("idx");
for d in [&dir0, &dir1, &idx] {
fs::create_dir_all(d).unwrap();
}
let collection = "c";
let vid = 4123u32;
plant_shard(&dir0, collection, vid, 2);
plant_shard(&dir1, collection, vid, 0);
fs::write(
dir0.join(format!("{}_{}.vif", collection, vid)),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: 1_700_000_000_000_000_043,
..Default::default()
}),
..Default::default()
};
fs::write(
dir1.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
store.store_dirs = vec![
dir0.to_str().unwrap().to_string(),
dir1.to_str().unwrap().to_string(),
];
for dir in [&dir0, &dir1] {
store
.add_location(
dir.to_str().unwrap(),
idx.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
for loc in &store.locations {
let ec_vol = loc
.find_ec_volume(VolumeId(vid))
.unwrap_or_else(|| panic!("{} ec volume mounted", loc.directory));
assert_eq!(
ec_vol.encode_ts_ns, 1_700_000_000_000_000_043,
"{}",
loc.directory
);
}
}
/// Starts a store whose first disk still holds the source `.dat`, its
/// config-free `.vif` and one 12+4 shard, with the `.ecx` in a shared
/// index dir and one sibling encode `.vif` per entry of `sibling_runs`.
/// Asserts the source `.dat`, the shard and the `.ecx` survive startup.
fn start_with_sibling_encode_vifs(sibling_runs: &[i64]) -> Store {
let tmp = TempDir::new().unwrap();
let dirs: Vec<_> = (0..=sibling_runs.len())
.map(|i| tmp.path().join(format!("data{}", i)))
.collect();
let idx = tmp.path().join("idx");
for d in dirs.iter().chain([&idx]) {
fs::create_dir_all(d).unwrap();
}
let collection = "c";
let vid = 4124u32;
let base0 = dirs[0].join(format!("{}_{}", collection, vid));
let dat = format!("{}.dat", base0.display());
let shard = format!("{}.ec02", base0.display());
fs::File::create(&dat)
.unwrap()
.set_len(11 * 1024 * 1024)
.unwrap();
fs::File::create(&shard)
.unwrap()
.set_len(1024 * 1024)
.unwrap();
fs::write(
format!("{}.vif", base0.display()),
r#"{"version":3,"readOnly":true}"#,
)
.unwrap();
plant_ecx(&idx, collection, vid, &[0xA1u8; 20]);
for (dir, run) in dirs[1..].iter().zip(sibling_runs) {
let vif = VifVolumeInfo {
version: 3,
ec_shard_config: Some(VifEcShardConfig {
data_shards: 12,
parity_shards: 4,
encode_ts_ns: *run,
..Default::default()
}),
..Default::default()
};
fs::write(
dir.join(format!("{}_{}.vif", collection, vid)),
serde_json::to_string(&vif).unwrap(),
)
.unwrap();
}
let mut store = Store::new(NeedleMapKind::InMemory);
store.store_dirs = dirs
.iter()
.map(|d| d.to_str().unwrap().to_string())
.collect();
for dir in &dirs {
store
.add_location(
dir.to_str().unwrap(),
idx.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
let ecx = idx.join(format!("{}_{}.ecx", collection, vid));
for path in [dat.as_str(), shard.as_str(), ecx.to_str().unwrap()] {
assert!(Path::new(path).exists(), "{} removed at startup", path);
}
store
}
/// A disk still holding the source `.dat` sizes its shards by the
/// sibling's encode `.vif` before the startup check decides whether they
/// are leftovers of an interrupted encode.
#[test]
fn shared_index_startup_validation_uses_sibling_vif_ratio() {
let store = start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044]);
let ec_vol = store.locations[0]
.find_ec_volume(VolumeId(4124))
.expect("dir0 ec volume mounted");
assert!(ec_vol.has_shard(2), "shard 2 not mounted on dir0");
}
/// Sibling `.vif` files naming different encode runs leave the ratio
/// unknown, so the startup check keeps the shards.
#[test]
fn shared_index_startup_validation_keeps_shards_on_disagreeing_siblings() {
start_with_sibling_encode_vifs(&[1_700_000_000_000_000_044, 1_700_000_000_000_000_045]);
}
#[test]
fn mirror_preserves_existing_destination_sidecars() {
let tmp = TempDir::new().unwrap();
@@ -701,7 +443,7 @@ mod tests {
add_loc(&mut store, &dir0);
add_loc(&mut store, &dir1);
let owners = store.index_ecx_owners_for_mirror(None);
let owners = store.index_ecx_owners_for_mirror();
let owner = owners
.get(&EcKey {
collection: collection.to_string(),
+18 -115
View File
@@ -92,15 +92,11 @@ impl Store {
///
/// Mirrors `Store.reconcileEcShardsAcrossDisks` in Go.
pub fn reconcile_ec_shards_across_disks(&mut self) {
self.reconcile_ec_shards_across_disks_only(None)
}
pub(crate) fn reconcile_ec_shards_across_disks_only(&mut self, only: Option<VolumeId>) {
if self.locations.len() < 2 {
return;
}
let owners = self.index_ecx_owners(only);
let owners = self.index_ecx_owners();
if owners.is_empty() {
return;
}
@@ -110,7 +106,7 @@ impl Store {
// loc.idx_directory instead of the owner disk.
let mut to_load: Vec<OrphanShardLoad> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let orphans = collect_orphan_ec_shards(loc, loc_idx, only);
let orphans = collect_orphan_ec_shards(loc, loc_idx);
for (key, shards) in orphans {
let Some(owner) = owners.get(&key) else {
warn!(
@@ -406,7 +402,7 @@ impl Store {
/// matched so downstream loaders point `EcVolume::new` at the
/// directory that really has the file. The first owner found wins;
/// duplicates across disks are unusual but tolerated.
fn index_ecx_owners(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwnerInfo> {
fn index_ecx_owners(&self) -> HashMap<EcKey, EcxOwnerInfo> {
let mut owners: HashMap<EcKey, EcxOwnerInfo> = HashMap::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
let mut seen: Vec<&str> = Vec::with_capacity(2);
@@ -426,18 +422,15 @@ impl Store {
let Some(base) = name.strip_suffix(".ecx") else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
// credible owner — skip it so the scan keeps looking for a
// real index on a sibling disk (Go's indexEcxOwners).
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
continue;
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
let key = EcKey { collection, vid };
owners.entry(key).or_insert(EcxOwnerInfo {
location: loc_idx,
@@ -457,18 +450,11 @@ impl Store {
///
/// Mirrors `Store.CollectEcVolumesMissingIndex` in Go.
pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec<EcVolumeMissingIndex> {
self.collect_ec_volumes_missing_index_only(None)
}
pub(crate) fn collect_ec_volumes_missing_index_only(
&self,
only: Option<VolumeId>,
) -> Vec<EcVolumeMissingIndex> {
let owners = self.index_ecx_owners(only);
let owners = self.index_ecx_owners();
let mut seen: HashSet<EcKey> = HashSet::new();
let mut missing = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) {
if owners.contains_key(&key) || !seen.insert(key.clone()) {
continue;
}
@@ -488,24 +474,18 @@ impl Store {
/// the disks that now have a local index, then fall back to the cross-disk
/// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go.
pub(crate) fn mount_recovered_ec_shards(&mut self) {
self.mount_recovered_ec_shards_only(None)
}
/// `mount_recovered_ec_shards` restricted to one volume's shards when
/// `only` names it.
pub(crate) fn mount_recovered_ec_shards_only(&mut self, only: Option<VolumeId>) {
self.mirror_ec_metadata_to_shard_disks_only(only);
self.load_orphan_ec_shards_with_local_index(only);
self.reconcile_ec_shards_across_disks_only(only);
self.mirror_ec_metadata_to_shard_disks();
self.load_orphan_ec_shards_with_local_index();
self.reconcile_ec_shards_across_disks();
}
/// Mount on-disk EC shards whose `.ecx` index is now present on the same disk.
/// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a
/// single-disk store recovers once its index has been fetched from a peer.
fn load_orphan_ec_shards_with_local_index(&mut self, only: Option<VolumeId>) {
fn load_orphan_ec_shards_with_local_index(&mut self) {
let mut work: Vec<(usize, EcKey, Vec<ShardId>)> = Vec::new();
for (loc_idx, loc) in self.locations.iter().enumerate() {
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) {
if !loc.has_ecx_file_on_disk(&key.collection, key.vid) {
continue;
}
@@ -536,7 +516,6 @@ impl Store {
fn collect_orphan_ec_shards(
loc: &crate::storage::disk_location::DiskLocation,
_loc_idx: usize,
only: Option<VolumeId>,
) -> HashMap<EcKey, Vec<(String, ShardId)>> {
let mut orphans: HashMap<EcKey, Vec<(String, ShardId)>> = HashMap::new();
let Ok(read) = fs::read_dir(&loc.directory) else {
@@ -554,12 +533,6 @@ fn collect_orphan_ec_shards(
let Some(shard_id) = is_ec_shard_extension(ext) else {
continue;
};
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
if only.is_some_and(|o| o != vid) {
continue;
}
// Ignore zero-byte shards. Use the DirEntry's metadata so we
// don't pay a second stat syscall per file beyond what
// read_dir already returned.
@@ -567,6 +540,9 @@ fn collect_orphan_ec_shards(
Ok(meta) if meta.len() > 0 => {}
_ => continue,
}
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
continue;
};
// Skip shards that are already registered to an EcVolume.
if let Some(ecv) = loc.find_ec_volume(vid)
&& ecv.has_shard(shard_id)
@@ -679,7 +655,7 @@ mod tests {
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
write_index_files(&d1, "pics", 7, 10, 4);
let owners = store.index_ecx_owners(None);
let owners = store.index_ecx_owners();
let owner = owners
.get(&EcKey {
collection: "pics".to_string(),
@@ -691,7 +667,7 @@ mod tests {
// A stub with no real index anywhere owns nothing.
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
assert!(!store.index_ecx_owners(None).contains_key(&EcKey {
assert!(!store.index_ecx_owners().contains_key(&EcKey {
collection: "pics".to_string(),
vid: VolumeId(8),
}));
@@ -1264,79 +1240,6 @@ mod tests {
);
}
#[test]
fn test_mount_recovered_ec_shards_only_named_volume() {
// Recovering one volume mounts only its shards. Another volume's orphan
// shards, such as ones an in-flight copy just renamed in, stay unmounted
// even when an index for them is already on disk. Mirrors Go
// TestMountRecoveredEcShards_OnlyNamedVolume.
let tmp = TempDir::new().unwrap();
let dir0 = tmp.path().join("data0");
let dir1 = tmp.path().join("data1");
std::fs::create_dir_all(&dir0).unwrap();
std::fs::create_dir_all(&dir1).unwrap();
let collection = "col";
let (recovered_vid, other_vid) = (71u32, 72u32);
let mut store = Store::new(NeedleMapKind::InMemory);
for d in [&dir0, &dir1] {
store
.add_location(
d.to_str().unwrap(),
d.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
for vid in [recovered_vid, other_vid] {
write_shard(dir0.to_str().unwrap(), collection, vid, 0);
write_shard(dir1.to_str().unwrap(), collection, vid, 1);
}
write_index_files(dir0.to_str().unwrap(), collection, other_vid, 10, 4);
let missing = store.collect_ec_volumes_missing_index();
let m = missing
.iter()
.find(|m| m.vid == VolumeId(recovered_vid))
.expect("recovered volume not reported as missing its index");
write_index_files(&m.idx_dir, collection, recovered_vid, 10, 4);
store.mount_recovered_ec_shards_only(Some(VolumeId(recovered_vid)));
for (i, loc) in store.locations.iter().enumerate() {
let ev = loc
.find_ec_volume(VolumeId(recovered_vid))
.unwrap_or_else(|| panic!("volume {} not mounted on disk{}", recovered_vid, i));
assert!(
ev.has_shard(i as u8),
"shard {}.{} not mounted",
recovered_vid,
i
);
assert!(
loc.find_ec_volume(VolumeId(other_vid)).is_none(),
"volume {} mounted on disk{} by the recovery of volume {}",
other_vid,
i,
recovered_vid,
);
}
let mirrored = format!(
"{}/{}_{}.ecx",
dir1.to_str().unwrap(),
collection,
other_vid
);
assert!(
!std::path::Path::new(&mirrored).exists(),
"volume {} index mirrored to disk1 by the recovery of volume {}",
other_vid,
recovered_vid,
);
}
#[test]
fn test_collect_missing_index_excludes_cross_disk_orphan() {
// A .ecx merely on a sibling disk is the same-server case the reconcile
File diff suppressed because it is too large. Load diff
@@ -188,7 +188,7 @@ mod tests {
for id in 1..=3 {
v.write_needle(&mut needle(id), true, false).unwrap();
}
v.delete_needle(&mut needle(2), false).unwrap();
v.delete_needle(&mut needle(2)).unwrap();
v.sync_to_disk().unwrap();
let (want_count, want_deleted) = (v.file_count(), v.deleted_count());
drop(v);
@@ -382,13 +382,10 @@ pub(crate) mod tests {
write_test_volume(dir, 6);
let v = open_volume(dir);
v.delete_needle(
&mut Needle {
id: NeedleId(3),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(3),
..Needle::default()
})
.unwrap();
drop(v);
@@ -420,13 +417,10 @@ pub(crate) mod tests {
write_test_volume(dir, 8);
let v = open_volume(dir);
v.delete_needle(
&mut Needle {
id: NeedleId(2),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(2),
..Needle::default()
})
.unwrap();
drop(v);
@@ -505,13 +505,10 @@ mod tests {
f.written.insert(*id, n.data.clone());
}
Op::Del(id) => {
v.delete_needle(
&mut Needle {
id: NeedleId(*id),
..Needle::default()
},
false,
)
v.delete_needle(&mut Needle {
id: NeedleId(*id),
..Needle::default()
})
.unwrap();
}
Op::Empty(id) => {
-118
View File
@@ -110,7 +110,6 @@ fn build_test_state(
vacuum_commit_slots: None,
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
@@ -1436,120 +1435,3 @@ async fn write_queue_upload_returns_same_etag_as_direct_write() {
}
assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
}
// ============================================================================
// A Range over a compressed needle is answered from the decoded bytes
// ============================================================================
#[tokio::test]
async fn range_with_accept_gzip_on_compressed_needle_serves_decoded_bytes() {
let (state, _tmp) = test_state();
let uri = "/1/03637037d6/notes.txt";
let plain: Vec<u8> = (0..400)
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
.collect();
let app = build_admin_router(state.clone());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::from(plain.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let read = |range: Option<&str>| {
let mut request = Request::builder()
.uri(uri)
.header("Accept-Encoding", "gzip");
if let Some(range) = range {
request = request.header("Range", range);
}
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
};
// The whole read passes the stored gzip through, so its length is the compressed size.
let response = read(None).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()["Content-Encoding"], "gzip");
let compressed_len = body_bytes(response).await.len();
assert!(compressed_len < plain.len() / 4);
for (start, end) in [(10, 69), (compressed_len + 10, compressed_len + 69)] {
let response = read(Some(&format!("bytes={start}-{end}"))).await.unwrap();
assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
assert!(response.headers().get("Content-Encoding").is_none());
assert_eq!(
response.headers()["Content-Range"],
format!("bytes {start}-{end}/{}", plain.len()).as_str()
);
assert_eq!(body_bytes(response).await, &plain[start..=end]);
}
}
// ============================================================================
// HEAD of a compressed needle reports the length the matching GET would send
// ============================================================================
#[tokio::test]
async fn head_on_compressed_needle_reports_served_length() {
let (state, _tmp) = test_state();
let uri = "/1/04637037d6/notes.txt";
let plain: Vec<u8> = (0..400)
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
.collect();
let app = build_admin_router(state.clone());
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(uri)
.body(Body::from(plain.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let send = |method: &str, accept_gzip: bool, range: Option<&str>| {
let mut request = Request::builder().method(method).uri(uri);
if accept_gzip {
request = request.header("Accept-Encoding", "gzip");
}
if let Some(range) = range {
request = request.header("Range", range);
}
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
};
let response = send("GET", true, None).await.unwrap();
let compressed_len = body_bytes(response).await.len();
assert!(compressed_len < plain.len() / 4);
for (accept_gzip, range, encoding, length) in [
(false, None, None, plain.len()),
(true, None, Some("gzip"), compressed_len),
(true, Some("bytes=0-9"), None, plain.len()),
] {
let response = send("HEAD", accept_gzip, range).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get("Content-Encoding")
.map(|v| v.to_str().unwrap()),
encoding,
"accept_gzip={accept_gzip} range={range:?}"
);
assert_eq!(
response.headers()["Content-Length"],
length.to_string().as_str(),
"accept_gzip={accept_gzip} range={range:?}"
);
}
}
-4
View File
@@ -7157,7 +7157,6 @@ dependencies = [
"arrow-array",
"arrow-cast",
"arrow-schema",
"arrow-select",
"async-trait",
"base64 0.22.1",
"bytes",
@@ -7171,10 +7170,8 @@ dependencies = [
"hmac 0.12.1",
"http 1.5.0",
"hyper-util",
"jsonb",
"jsonwebtoken",
"lance",
"lance-arrow",
"lance-datafusion",
"lance-index",
"lance-linalg",
@@ -7206,7 +7203,6 @@ dependencies = [
"tower",
"tracing",
"tracing-subscriber",
"uuid",
"x509-parser",
"zstd",
]
+2 -39
View File
@@ -3,7 +3,7 @@ use tokio::sync::mpsc;
use crate::pb::{
ActivityEvent, DetectionComplete, DetectionProposals, JobCompleted, JobProgressUpdate,
JobState, WorkerObservations, WorkerToAdminMessage, worker_to_admin_message::Body,
WorkerObservations, WorkerToAdminMessage, worker_to_admin_message::Body,
};
/// Replies to one detection request.
@@ -65,12 +65,7 @@ impl DetectionSender for StreamSender {
}
impl ExecutionSender for StreamSender {
fn send_progress(&self, mut progress: JobProgressUpdate) -> Result<()> {
// the Go handlers always report a state; older admins read a missing
// one as an ended job and drop its completion
if progress.state == JobState::Unspecified as i32 {
progress.state = JobState::Running as i32;
}
fn send_progress(&self, progress: JobProgressUpdate) -> Result<()> {
self.send(Body::JobProgressUpdate(progress))
}
@@ -205,36 +200,4 @@ mod tests {
.expect("send");
assert!(metered.reported_failure());
}
#[test]
fn progress_without_a_state_is_sent_as_running() {
let (tx, mut rx) = mpsc::unbounded_channel();
let stream = StreamSender::new("worker-1".to_string(), tx);
let metered = MeteredSender::new(&stream);
metered
.send_progress(JobProgressUpdate {
progress_percent: 10.0,
..Default::default()
})
.expect("send");
metered
.send_progress(JobProgressUpdate {
state: JobState::Failed as i32,
..Default::default()
})
.expect("send");
let mut states = Vec::new();
while let Ok(message) = rx.try_recv() {
let Some(Body::JobProgressUpdate(progress)) = message.body else {
panic!("expected a progress update");
};
states.push(progress.state);
}
assert_eq!(
states,
vec![JobState::Running as i32, JobState::Failed as i32]
);
}
}
+1 -8
View File
@@ -35,16 +35,10 @@ lance-table = "10"
# For the execution options the sort needs; the scanner's own stream helper
# takes the default, which has spilling off.
lance-datafusion = "10"
arrow-array = "58"
arrow-schema = "58"
arrow-cast = "58"
arrow-select = "58"
# Lance's stored form of the metadata JSON, read when deriving the
# indexed-key columns.
jsonb = { version = "0.5", default-features = false }
chrono = "0.4"
futures = "0.3"
uuid = "1"
anyhow.workspace = true
async-trait.workspace = true
clap = { version = "4", features = ["derive", "env"] }
@@ -94,10 +88,9 @@ protoc-bin-vendored = "3"
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tonic.workspace = true
lance-arrow = "10"
arrow-array = "58"
arrow-schema = "58"
arrow-cast = "58"
arrow-select = "58"
lance-linalg = "10"
tempfile = "3"
+2 -18
View File
@@ -18,8 +18,7 @@ use lance::io::{
use lance::session::Session;
use crate::catalog::{NamespaceClient, TableDescription};
use crate::direct_read::{DirectReads, Wrappers};
use crate::drop_guard::DropGuard;
use crate::direct_read::DirectReads;
/// A table the worker is about to work on.
pub struct OpenTable {
@@ -76,7 +75,7 @@ pub fn unsigned_options() -> HashMap<String, String> {
pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) -> Result<OpenTable> {
let description = client.describe_table(id).await?;
let wrapper = storage.wrapper(&description.location).await?;
let dataset = open_guarded(&description, wrapper).await?;
let dataset = open_at(&description, wrapper, None).await?;
Ok(OpenTable {
id: id.to_vec(),
location: description.location,
@@ -84,21 +83,6 @@ pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) ->
})
}
/// [`open_at`] for a job that writes the table: its writes fail once the
/// table is dropped under it (see [`DropGuard`]).
pub async fn open_guarded(
description: &TableDescription,
wrapper: Option<Arc<dyn WrappingObjectStore>>,
) -> Result<Dataset> {
let guard = DropGuard::new();
let mut wrappers: Vec<Arc<dyn WrappingObjectStore>> = wrapper.into_iter().collect();
wrappers.push(guard.clone());
let dataset = open_at(description, Some(Arc::new(Wrappers(wrappers))), None).await?;
let opened = dataset.manifest_location();
guard.arm(opened.path.clone(), opened.e_tag.clone());
Ok(dataset)
}
/// Open a table the caller has already resolved, with the store wrapped, in
/// `session` when given, otherwise in a session of its own.
pub async fn open_at(
@@ -1,322 +0,0 @@
//! Keeps a job from writing into a table dropped under it.
//!
//! DeleteIndex purges a table's directory, and an index created again under
//! the same name reuses it. A job that opened the table before the purge
//! would otherwise go on writing there: its data files would land in the
//! new table's directory, and its commit would put a manifest of the old
//! table there, so the new table could not be created, or would read it.
use std::fmt;
use std::ops::Range;
use std::sync::{Arc, Mutex};
use bytes::Bytes;
use futures::StreamExt;
use futures::stream::BoxStream;
use lance::io::WrappingObjectStore;
use object_store::path::Path;
use object_store::{
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
UploadPart,
};
/// Fails every write once the manifest it watches is gone or is another
/// object (a table created again reuses version paths): the one of the
/// version the job opened, then the newest the job committed. Armed after the
/// open; until then it lets everything through. When the job itself deletes
/// the watched manifest, removing old versions as a purge does, it watches
/// the newest manifest left instead.
#[derive(Debug, Default)]
pub struct DropGuard {
watched: Arc<Mutex<Watched>>,
}
#[derive(Debug, Default, Clone)]
struct Watched {
manifest: Option<Path>,
/// The manifest's e-tag when known, telling it from a replacement.
e_tag: Option<String>,
/// The job deleted the watched manifest itself.
deleted: bool,
}
impl DropGuard {
pub fn new() -> Arc<Self> {
Arc::new(Self::default())
}
/// Watch `manifest`, the opened version's, with its e-tag when known.
pub fn arm(&self, manifest: Path, e_tag: Option<String>) {
let mut watched = self.watched.lock().unwrap();
watched.manifest = Some(manifest);
watched.e_tag = e_tag;
}
}
impl WrappingObjectStore for DropGuard {
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
Arc::new(Guarded {
inner: original,
watched: self.watched.clone(),
})
}
}
#[derive(Debug, Clone)]
struct Guarded {
inner: Arc<dyn ObjectStore>,
watched: Arc<Mutex<Watched>>,
}
/// What a check found: the table is there, or it was dropped, or created
/// again, with the error to fail the write with.
enum Found {
Table,
Dropped(object_store::Error),
Replaced(object_store::Error),
}
impl Guarded {
/// Whether the watched manifest is still the table's. An error is a
/// failed probe, which says nothing about the table.
async fn find(&self, location: &Path) -> Result<Found> {
let watched = self.watched.lock().unwrap().clone();
let Some(mut manifest) = watched.manifest else {
return Ok(Found::Table);
};
let mut e_tag = watched.e_tag;
if watched.deleted {
if let Some((newest, tag)) = self.newest_manifest(&manifest).await? {
manifest = newest;
e_tag = tag;
}
let mut watched = self.watched.lock().unwrap();
watched.manifest = Some(manifest.clone());
watched.e_tag = e_tag.clone();
watched.deleted = false;
}
let error = |why: &str| {
object_store::Error::Generic {
store: "DropGuard",
source: format!(
"the table was dropped while this job wrote it: {manifest} {why}, so {location} is not written"
)
.into(),
}
};
match self.inner.head(&manifest).await {
Ok(meta) => match (&e_tag, &meta.e_tag) {
(Some(want), Some(have)) if want != have => {
Ok(Found::Replaced(error("is another table's")))
}
_ => Ok(Found::Table),
},
// Gone with manifests beside it: a table created again, at an
// earlier version than the one watched.
Err(object_store::Error::NotFound { .. }) => {
match self.newest_manifest(&manifest).await? {
Some(_) => Ok(Found::Replaced(error("is gone and another table is there"))),
None => Ok(Found::Dropped(error("is gone"))),
}
}
Err(e) => Err(e),
}
}
async fn check(&self, location: &Path) -> Result<()> {
match self.find(location).await? {
Found::Table => Ok(()),
Found::Dropped(error) | Found::Replaced(error) => Err(error),
}
}
}
fn is_manifest(location: &Path) -> bool {
location.extension() == Some("manifest")
}
impl Guarded {
/// The most recently written manifest beside `manifest`, with its e-tag.
async fn newest_manifest(&self, manifest: &Path) -> Result<Option<(Path, Option<String>)>> {
let dir: Path = manifest
.parts()
.take(manifest.parts().count() - 1)
.collect();
let listed = self.inner.list_with_delimiter(Some(&dir)).await?;
Ok(listed
.objects
.into_iter()
.filter(|o| is_manifest(&o.location))
.max_by_key(|o| o.last_modified)
.map(|o| (o.location, o.e_tag)))
}
fn committed(&self, location: &Path, e_tag: Option<String>) {
if is_manifest(location) {
let mut watched = self.watched.lock().unwrap();
if watched.manifest.is_some() {
watched.manifest = Some(location.clone());
watched.e_tag = e_tag;
watched.deleted = false;
}
}
}
}
/// A multipart upload that checks the table again before it completes.
#[derive(Debug)]
struct GuardedUpload {
inner: Box<dyn MultipartUpload>,
guard: Guarded,
location: Path,
}
#[async_trait::async_trait]
impl MultipartUpload for GuardedUpload {
fn put_part(&mut self, data: PutPayload) -> UploadPart {
self.inner.put_part(data)
}
async fn complete(&mut self) -> Result<PutResult> {
// A failed probe aborts too: the parts would outlive the upload.
let error = match self.guard.find(&self.location).await {
Ok(Found::Table) => return self.inner.complete().await,
Ok(Found::Dropped(error) | Found::Replaced(error)) | Err(error) => error,
};
let _ = self.inner.abort().await;
Err(error)
}
async fn abort(&mut self) -> Result<()> {
self.inner.abort().await
}
}
impl fmt::Display for Guarded {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "DropGuard({})", self.inner)
}
}
#[async_trait::async_trait]
impl ObjectStore for Guarded {
async fn put_opts(
&self,
location: &Path,
payload: PutPayload,
opts: PutOptions,
) -> Result<PutResult> {
self.check(location).await?;
let put = self.inner.put_opts(location, payload, opts).await?;
// A drop can land between the check and the write. A manifest is
// what makes the table readable again, so look once more and take
// it back if the table is gone; a failed probe leaves it. A table
// created again since may have written its own manifest there:
// only a manifest known to be this write's is taken back.
if is_manifest(location) {
let (error, replaced) = match self.find(location).await {
Ok(Found::Dropped(error)) => (error, false),
Ok(Found::Replaced(error)) => (error, true),
_ => {
self.committed(location, put.e_tag.clone());
return Ok(put);
}
};
let ours = match (&put.e_tag, self.inner.head(location).await) {
(Some(wrote), Ok(meta)) => meta.e_tag.as_ref() == Some(wrote),
(_, Err(object_store::Error::NotFound { .. })) => false,
_ => !replaced,
};
if ours {
let _ = self.inner.delete(location).await;
}
return Err(error);
}
self.committed(location, put.e_tag.clone());
Ok(put)
}
async fn put_multipart_opts(
&self,
location: &Path,
opts: PutMultipartOptions,
) -> Result<Box<dyn MultipartUpload>> {
self.check(location).await?;
let inner = self.inner.put_multipart_opts(location, opts).await?;
Ok(Box::new(GuardedUpload {
inner,
guard: self.clone(),
location: location.clone(),
}))
}
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
self.inner.get_opts(location, options).await
}
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
self.inner.get_ranges(location, ranges).await
}
fn delete_stream(
&self,
locations: BoxStream<'static, Result<Path>>,
) -> BoxStream<'static, Result<Path>> {
// A cleanup picks its paths before it deletes them; a table created
// again meanwhile reuses those paths, so look once before deleting.
// In a dropped table there is nothing left to spare, and the job's
// own failed commit is cleaned up.
let watched = self.watched.clone();
let locations = locations.inspect(move |location| {
if let Ok(location) = location {
let mut watched = watched.lock().unwrap();
if watched.manifest.as_ref() == Some(location) {
watched.deleted = true;
}
}
});
let guard = self.clone();
let pending = Arc::new(Mutex::new(Some(locations.boxed())));
let checked =
futures::stream::once(async move { guard.find(&Path::from("cleanup")).await })
.flat_map(move |found| {
let locations = pending.lock().unwrap().take();
match (found, locations) {
(Ok(Found::Table | Found::Dropped(_)), Some(locations)) => locations,
(Ok(Found::Replaced(error)), _) | (Err(error), _) => {
futures::stream::once(async move { Err(error) }).boxed()
}
(Ok(_), None) => futures::stream::empty().boxed(),
}
})
.boxed();
self.inner.delete_stream(checked)
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list(prefix)
}
fn list_with_offset(
&self,
prefix: Option<&Path>,
offset: &Path,
) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list_with_offset(prefix, offset)
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
self.inner.list_with_delimiter(prefix).await
}
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
self.check(to).await?;
self.inner.copy_opts(from, to, options).await
}
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
self.check(to).await?;
self.inner.rename_opts(from, to, options).await
}
}
@@ -1,701 +0,0 @@
//! A table dropped while a worker job runs on it: the job must fail without
//! writing into the dropped path, and a table created again there starts
//! empty.
use std::fmt;
use std::ops::Range;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use bytes::Bytes;
use futures::StreamExt;
use futures::stream::BoxStream;
use lance::Dataset;
use lance::io::WrappingObjectStore;
use object_store::path::Path;
use object_store::{
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
};
use crate::metadata_index::tests::lifecycle::{table, upsert};
use crate::metadata_index::{self, State};
/// Removes the table directory just before the first write through it, the
/// way DeleteIndex purging the table between a job's open and its writes
/// would.
#[derive(Debug)]
struct DropOnFirstWrite {
dir: PathBuf,
fired: Arc<AtomicBool>,
}
impl WrappingObjectStore for DropOnFirstWrite {
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
Arc::new(Dropping {
inner: original,
dir: self.dir.clone(),
fired: self.fired.clone(),
})
}
}
#[derive(Debug)]
struct Dropping {
inner: Arc<dyn ObjectStore>,
dir: PathBuf,
fired: Arc<AtomicBool>,
}
impl Dropping {
fn drop_once(&self) {
if !self.fired.swap(true, Ordering::SeqCst) {
std::fs::remove_dir_all(&self.dir).unwrap();
}
}
}
impl fmt::Display for Dropping {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Dropping({})", self.inner)
}
}
#[async_trait::async_trait]
impl ObjectStore for Dropping {
async fn put_opts(
&self,
location: &Path,
payload: PutPayload,
opts: PutOptions,
) -> Result<PutResult> {
self.drop_once();
self.inner.put_opts(location, payload, opts).await
}
async fn put_multipart_opts(
&self,
location: &Path,
opts: PutMultipartOptions,
) -> Result<Box<dyn MultipartUpload>> {
self.drop_once();
self.inner.put_multipart_opts(location, opts).await
}
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
self.inner.get_opts(location, options).await
}
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
self.inner.get_ranges(location, ranges).await
}
fn delete_stream(
&self,
locations: BoxStream<'static, Result<Path>>,
) -> BoxStream<'static, Result<Path>> {
self.inner.delete_stream(locations)
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list(prefix)
}
fn list_with_offset(
&self,
prefix: Option<&Path>,
offset: &Path,
) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list_with_offset(prefix, offset)
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
self.inner.list_with_delimiter(prefix).await
}
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
self.inner.copy_opts(from, to, options).await
}
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
self.inner.rename_opts(from, to, options).await
}
}
/// Opens the table the way a job does; `during` drops it at the job's first
/// write, otherwise it is dropped right after the open.
async fn open_then_drop(dir: &tempfile::TempDir, during: bool) -> Dataset {
let description = crate::catalog::TableDescription {
location: dir.path().to_str().unwrap().to_string(),
storage_options: Default::default(),
managed_versioning: false,
};
let dropping: Option<Arc<dyn WrappingObjectStore>> = during.then(|| {
Arc::new(DropOnFirstWrite {
dir: dir.path().to_path_buf(),
fired: Arc::new(AtomicBool::new(false)),
}) as Arc<dyn WrappingObjectStore>
});
let dataset = crate::dataset::open_guarded(&description, dropping)
.await
.unwrap();
if !during {
std::fs::remove_dir_all(dir.path()).unwrap();
}
dataset
}
fn files(dir: &std::path::Path) -> Vec<String> {
let mut out = Vec::new();
if let Ok(entries) = std::fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
out.extend(files(&path));
} else {
out.push(path.display().to_string());
}
}
}
out
}
/// Runs `job` on a table dropped right after the job opens it, or at the
/// job's first write. The job must fail, leave no manifest (nor, dropped
/// before it wrote, any file) in the dropped path, and a table created
/// there again must start empty.
async fn dropped_during<F, Fut>(
label: &str,
during: bool,
prepare: impl AsyncFnOnce(&mut Dataset),
job: F,
) where
F: FnOnce(Dataset) -> Fut,
Fut: std::future::Future<Output = anyhow::Result<()>>,
{
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 4).await;
prepare(&mut dataset).await;
let result = job(open_then_drop(&dir, during).await).await;
let left = files(dir.path());
let error = result.expect_err(&format!(
"{label}: a job on a dropped table succeeded, leaving {left:?}"
));
eprintln!(
"{label} (dropped {}): {error:#}",
if during {
"at its first write"
} else {
"after the open"
}
);
assert!(
!left.iter().any(|f| f.ends_with(".manifest")),
"{label}: a manifest was written into the dropped table: {left:?}"
);
if !during {
assert!(
left.is_empty(),
"{label}: files were written into the dropped table: {left:?}"
);
}
let recreated = Dataset::write(
arrow_array::RecordBatchIterator::new(
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
),
dir.path().to_str().unwrap(),
None,
)
.await
.unwrap_or_else(|e| panic!("{label}: the table could not be created again: {e}"));
assert_eq!(recreated.version().version, 1, "{label}");
assert_eq!(recreated.count_rows(None).await.unwrap(), 0, "{label}");
}
/// Every job, with the table dropped after its open and at its first write.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_job_on_a_dropped_table_fails_and_leaves_it_dropped() {
for during in [false, true] {
dropped_during(
"vector index build",
during,
async |_| {},
async |mut d| {
crate::vector_index::build(
&mut d,
"vector",
crate::vector_index::INDEX_NAME,
4,
false,
)
.await
},
)
.await;
dropped_during(
"metadata key add",
during,
async |d| metadata_index::request(d, "author").await.unwrap(),
async |mut d| {
let key = metadata_index::add_column(&mut d, "author").await?;
metadata_index::finish_after(&mut d, &key, 1_000, std::time::Duration::ZERO)
.await
.map(|_| ())
},
)
.await;
dropped_during(
"metadata key maintenance",
during,
async |d| {
let key = metadata_index::add_column(d, "author").await.unwrap();
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
.await
.unwrap();
*d = upsert(d.clone(), 4_000..4_500).await;
},
async |mut d| {
let key = metadata_index::keys_of(&d).remove(0);
metadata_index::maintain(&mut d, &key, 1_000, 0)
.await
.map(|_| ())
},
)
.await;
dropped_during(
"metadata key remove",
during,
async |d| {
let key = metadata_index::add_column(d, "author").await.unwrap();
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
.await
.unwrap();
metadata_index::set_state(d, "_mk_0", State::Off)
.await
.unwrap();
},
async |mut d| metadata_index::remove_column(&mut d, "_mk_0").await,
)
.await;
}
}
/// A job that removes old versions itself, as a purge does, removes the
/// manifest it opened: that is no drop, and its later writes go through.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_purge_through_the_guard_completes() {
let dir = tempfile::TempDir::new().unwrap();
let dataset = table(&dir, 4).await;
let mut dataset = upsert(dataset, 0..100).await;
dataset.delete("key = 'k1' OR key = 'k2001'").await.unwrap();
let description = crate::catalog::TableDescription {
location: dir.path().to_str().unwrap().to_string(),
storage_options: Default::default(),
managed_versioning: false,
};
let mut guarded = crate::dataset::open_guarded(&description, None)
.await
.unwrap();
// The job commits, then removes every older version, the one it opened
// with them.
guarded
.update_config([("seaweedfs.test", "1")])
.await
.unwrap();
let removed = lance::dataset::cleanup::cleanup_old_versions(
&guarded,
lance::dataset::cleanup::CleanupPolicy {
before_timestamp: Some(chrono::Utc::now() + chrono::Duration::hours(1)),
before_version: Some(guarded.version().version),
..Default::default()
},
)
.await
.unwrap();
assert!(removed.old_versions > 0);
guarded
.update_config([("seaweedfs.test", "2")])
.await
.unwrap();
crate::jobs::purge::purge(&mut guarded, "t1").await.unwrap();
assert_eq!(guarded.count_rows(None).await.unwrap(), 3_998);
}
/// The table is dropped and created again up to the version the job opened,
/// so the manifest path the job watches exists again, as another object:
/// the job must still fail and leave the new table as it was.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_job_on_a_table_created_again_fails() {
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 4).await;
let key = metadata_index::add_column(&mut dataset, "author")
.await
.unwrap();
metadata_index::finish_after(&mut dataset, &key, 1_000, std::time::Duration::ZERO)
.await
.unwrap();
metadata_index::set_state(&mut dataset, "_mk_0", State::Off)
.await
.unwrap();
let description = crate::catalog::TableDescription {
location: dir.path().to_str().unwrap().to_string(),
storage_options: Default::default(),
managed_versioning: false,
};
let mut job = crate::dataset::open_guarded(&description, None)
.await
.unwrap();
let opened = job.version().version;
std::fs::remove_dir_all(dir.path()).unwrap();
let mut recreated = table(&dir, 1).await;
while recreated.version().version < opened {
let version = recreated.version().version.to_string();
recreated
.update_config([("seaweedfs.test", version.as_str())])
.await
.unwrap();
}
assert_eq!(recreated.version().version, opened);
let error = metadata_index::remove_column(&mut job, "_mk_0")
.await
.expect_err("a job on a table created again succeeded");
assert!(format!("{error:#}").contains("DropGuard"), "{error:#}");
recreated.checkout_latest().await.unwrap();
assert_eq!(recreated.version().version, opened);
assert_eq!(recreated.count_rows(None).await.unwrap(), 1_000);
}
fn guarded_description(dir: &tempfile::TempDir) -> crate::catalog::TableDescription {
crate::catalog::TableDescription {
location: dir.path().to_str().unwrap().to_string(),
storage_options: Default::default(),
managed_versioning: false,
}
}
/// The table's directory in the store's own paths.
fn table_root(dataset: &Dataset) -> Path {
let manifest = &dataset.manifest_location().path;
manifest
.parts()
.take(manifest.parts().count() - 2)
.collect()
}
/// A multipart upload started before the drop does not complete after it.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_multipart_upload_does_not_complete_into_a_dropped_table() {
let dir = tempfile::TempDir::new().unwrap();
table(&dir, 1).await;
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like::default());
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
.await
.unwrap();
let store = job.object_store(None).await.unwrap().inner.clone();
let path = table_root(&job).child("data").child("late.lance");
let mut upload = store.put_multipart(&path).await.unwrap();
upload
.put_part(PutPayload::from(vec![0u8; 16]))
.await
.unwrap();
std::fs::remove_dir_all(dir.path()).unwrap();
let completed = upload.complete().await;
assert!(
completed.is_err(),
"an upload completed into a dropped table"
);
assert!(files(dir.path()).is_empty(), "{:?}", files(dir.path()));
}
/// An upload whose last probe fails is aborted, not left with its parts.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_failed_probe_aborts_an_upload() {
let dir = tempfile::TempDir::new().unwrap();
table(&dir, 1).await;
let s3_like = S3Like::default();
let (armed, aborted) = (s3_like.armed.clone(), s3_like.aborted.clone());
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(s3_like);
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
.await
.unwrap();
let store = job.object_store(None).await.unwrap().inner.clone();
let path = table_root(&job).child("data").child("late.lance");
let mut upload = store.put_multipart(&path).await.unwrap();
upload
.put_part(PutPayload::from(vec![0u8; 16]))
.await
.unwrap();
armed.store(true, Ordering::SeqCst);
assert!(upload.complete().await.is_err());
assert!(aborted.load(Ordering::SeqCst), "the upload was left open");
}
/// A table created again, through the version a job just wrote, between
/// that write and its check keeps its own manifest at that path.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_rollback_keeps_the_manifest_of_a_table_created_again() {
let dir = tempfile::TempDir::new().unwrap();
table(&dir, 1).await;
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
..Default::default()
});
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
.await
.unwrap();
assert_eq!(job.version().version, 1);
assert!(
job.update_config([("seaweedfs.test", "job")])
.await
.is_err()
);
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
assert_eq!(again.version().version, 2);
assert_eq!(
again.config().get("seaweedfs.test").map(String::as_str),
Some("again")
);
}
/// A table created again whose manifest at the job's path cannot be told
/// from the job's, because the HEAD fails, keeps it.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_failed_rollback_probe_keeps_the_manifest_of_a_table_created_again() {
let dir = tempfile::TempDir::new().unwrap();
table(&dir, 1).await;
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
fail_head_after_manifest: true,
heads_before_failing: 1,
..Default::default()
});
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
.await
.unwrap();
assert!(
job.update_config([("seaweedfs.test", "job")])
.await
.is_err()
);
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
assert_eq!(again.version().version, 2);
assert_eq!(
again.config().get("seaweedfs.test").map(String::as_str),
Some("again")
);
}
/// A cleanup that picked its paths before the drop does not delete the files
/// of the table created again there.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_cleanup_does_not_delete_a_table_created_again() {
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 1).await;
dataset
.update_config([("seaweedfs.test", "1")])
.await
.unwrap();
let job = crate::dataset::open_guarded(&guarded_description(&dir), None)
.await
.unwrap();
let store = job.object_store(None).await.unwrap().inner.clone();
// The old version-1 manifest, as a cleanup would have picked it.
let mut first = dataset.checkout_version(1).await.unwrap();
let old = first.manifest_location().path.clone();
std::fs::remove_dir_all(dir.path()).unwrap();
let recreated = table(&dir, 1).await;
assert_eq!(recreated.manifest_location().path, old);
let deleted: Vec<_> = store
.delete_stream(futures::stream::iter(vec![Ok(old.clone())]).boxed())
.collect()
.await;
assert!(deleted.iter().any(|r| r.is_err()), "{deleted:?}");
first = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
assert_eq!(first.version().version, 1);
}
/// A probe that fails after a manifest was written says nothing about the
/// table: the manifest stays and the commit succeeds.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_failed_probe_does_not_take_back_a_commit() {
let dir = tempfile::TempDir::new().unwrap();
table(&dir, 1).await;
let flaky: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
fail_head_after_manifest: true,
..Default::default()
});
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(flaky))
.await
.unwrap();
let before = job.version().version;
job.update_config([("seaweedfs.test", "1")]).await.unwrap();
let reopened = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
assert_eq!(reopened.version().version, before + 1);
}
/// Behaves as an object store, not a file system: a multipart upload holds
/// its parts until it completes, and records an abort. Optionally fails one
/// HEAD after a manifest is written, as a timeout would, or creates the table
/// again, through two versions, right after the first manifest is written.
#[derive(Debug, Default)]
struct S3Like {
fail_head_after_manifest: bool,
/// HEADs let through after a manifest is written before one fails.
heads_before_failing: usize,
/// The next HEAD fails.
armed: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
}
impl WrappingObjectStore for S3Like {
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
Arc::new(S3LikeStore {
inner: original,
fail_head_after_manifest: self.fail_head_after_manifest,
heads_before_failing: self.heads_before_failing,
heads_left: Arc::new(AtomicUsize::new(0)),
armed: self.armed.clone(),
aborted: self.aborted.clone(),
recreate: self.recreate.clone(),
})
}
}
#[derive(Debug)]
struct S3LikeStore {
inner: Arc<dyn ObjectStore>,
fail_head_after_manifest: bool,
heads_before_failing: usize,
heads_left: Arc<AtomicUsize>,
armed: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
}
impl fmt::Display for S3LikeStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "S3Like({})", self.inner)
}
}
#[async_trait::async_trait]
impl ObjectStore for S3LikeStore {
async fn put_opts(
&self,
location: &Path,
payload: PutPayload,
opts: PutOptions,
) -> Result<PutResult> {
let put = self.inner.put_opts(location, payload, opts).await?;
if location.extension() == Some("manifest") {
if self.fail_head_after_manifest {
self.heads_left
.store(self.heads_before_failing, Ordering::SeqCst);
self.armed.store(true, Ordering::SeqCst);
}
if let Some((dir, fired)) = &self.recreate
&& !fired.swap(true, Ordering::SeqCst)
{
std::fs::remove_dir_all(dir).unwrap();
let mut again = create_empty(dir).await;
again
.update_config([("seaweedfs.test", "again")])
.await
.unwrap();
}
}
Ok(put)
}
async fn put_multipart_opts(
&self,
location: &Path,
_opts: PutMultipartOptions,
) -> Result<Box<dyn MultipartUpload>> {
Ok(Box::new(HeldParts {
inner: self.inner.clone(),
location: location.clone(),
parts: Vec::new(),
aborted: self.aborted.clone(),
}))
}
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
if options.head
&& self.armed.load(Ordering::SeqCst)
&& self
.heads_left
.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
.is_err()
&& self.armed.swap(false, Ordering::SeqCst)
{
return Err(object_store::Error::Generic {
store: "S3Like",
source: "timed out".into(),
});
}
self.inner.get_opts(location, options).await
}
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
self.inner.get_ranges(location, ranges).await
}
fn delete_stream(
&self,
locations: BoxStream<'static, Result<Path>>,
) -> BoxStream<'static, Result<Path>> {
self.inner.delete_stream(locations)
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list(prefix)
}
fn list_with_offset(
&self,
prefix: Option<&Path>,
offset: &Path,
) -> BoxStream<'static, Result<ObjectMeta>> {
self.inner.list_with_offset(prefix, offset)
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
self.inner.list_with_delimiter(prefix).await
}
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
self.inner.copy_opts(from, to, options).await
}
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
self.inner.rename_opts(from, to, options).await
}
}
/// Parts held outside the table until the upload completes, as S3 does.
#[derive(Debug)]
struct HeldParts {
inner: Arc<dyn ObjectStore>,
location: Path,
parts: Vec<Bytes>,
aborted: Arc<AtomicBool>,
}
#[async_trait::async_trait]
impl MultipartUpload for HeldParts {
fn put_part(&mut self, data: PutPayload) -> object_store::UploadPart {
self.parts.extend(data);
Box::pin(async { Ok(()) })
}
async fn complete(&mut self) -> Result<PutResult> {
let payload: PutPayload = self.parts.drain(..).collect();
self.inner.put(&self.location, payload).await
}
async fn abort(&mut self) -> Result<()> {
self.parts.clear();
self.aborted.store(true, Ordering::SeqCst);
Ok(())
}
}
/// An empty table at `dir`.
async fn create_empty(dir: &std::path::Path) -> Dataset {
Dataset::write(
arrow_array::RecordBatchIterator::new(
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
),
dir.to_str().unwrap(),
None,
)
.await
.unwrap()
}
+30 -163
View File
@@ -3,12 +3,11 @@ use std::collections::HashMap;
use anyhow::{Context, Result, anyhow};
use async_trait::async_trait;
use chrono::{Duration, Utc};
use lance::Dataset;
use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions};
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
use seaweed_worker_core::pb::{
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
RunDetectionRequest,
};
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
@@ -16,7 +15,6 @@ use tracing::warn;
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::purge::{self, Retention};
use crate::jobs::{clamp, string_list, table_id};
pub const JOB_TYPE: &str = "lance_cleanup_versions";
@@ -30,9 +28,7 @@ const MAX_MIN_VERSIONS: i64 = 1000;
/// Lance keeps every version until something removes it. Lance can also do this
/// itself through auto-cleanup, so this job is for deployments that would rather
/// the cluster owned the policy than each writer. A table may override the
/// retention in its own config, and may ask for a purge (see [`purge`]), which
/// this job carries out ahead of the retention window.
/// the cluster owned the policy than each writer.
pub struct CleanupVersionsHandler {
namespace_url: String,
storage: dataset::Storage,
@@ -72,8 +68,9 @@ impl CleanupVersionsHandler {
/// The oldest version that may be removed while still leaving `min_versions`
/// behind, or None when the table has no more than the floor.
async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<u64>> {
let mut versions: Vec<u64> = dataset
async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result<Option<u64>> {
let mut versions: Vec<u64> = table
.dataset
.versions()
.await?
.iter()
@@ -86,33 +83,6 @@ async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<
Ok(Some(versions[versions.len() - min_versions]))
}
/// The cleanup that keeps `min_versions` and every version younger than
/// `retain_hours`, or None when the floor leaves nothing to remove. A purge
/// cleans up through it too.
pub async fn retention_policy(
dataset: &Dataset,
retain_hours: i64,
min_versions: usize,
) -> Result<Option<CleanupPolicy>> {
let Some(floor) = version_floor(dataset, min_versions).await? else {
return Ok(None);
};
Ok(Some(CleanupPolicy {
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
// should_clean() ANDs its clauses, so a version has to be both older
// than the window and below the floor to go.
before_version: Some(floor),
// Files this dataset cannot account for are left alone: they may
// belong to a writer that has not committed yet, and deleting them
// would corrupt a commit in flight.
delete_unverified: false,
// A tagged version is pinned on purpose, so refuse rather than
// silently dropping what someone named.
error_if_tagged_old_versions: true,
..Default::default()
}))
}
#[async_trait]
impl JobHandler for CleanupVersionsHandler {
fn capability(&self) -> JobTypeCapability {
@@ -164,12 +134,6 @@ impl JobHandler for CleanupVersionsHandler {
defaults.clone(),
)),
worker_default_values: defaults,
// A purge compacts and retrains, so it gets a build's time.
admin_runtime_defaults: Some(AdminRuntimeDefaults {
job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
..Default::default()
}),
..Default::default()
}
}
@@ -217,23 +181,10 @@ impl JobHandler for CleanupVersionsHandler {
continue;
}
};
let config = table.dataset.config();
let pending_purge = purge::pending(config);
let floor = Retention::for_table(
config,
Retention {
retain_hours: DEFAULT_RETAIN_HOURS,
min_versions,
},
MAX_RETAIN_HOURS,
MAX_MIN_VERSIONS,
)
.min_versions;
// Age is decided at execution against the retention window; a table
// at or under the floor cannot lose a version whatever its age, so
// proposing one would only produce a job with nothing to do. A
// requested purge ignores the floor: it keeps only the latest.
if pending_purge.is_none() && stats.total_versions <= floor {
// proposing one would only produce a job with nothing to do.
if stats.total_versions <= min_versions {
continue;
}
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
@@ -242,21 +193,12 @@ impl JobHandler for CleanupVersionsHandler {
proposal_id: format!("{JOB_TYPE}:{encoded}"),
dedupe_key: format!("{JOB_TYPE}:{encoded}"),
job_type: JOB_TYPE.to_string(),
summary: match &pending_purge {
Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions),
None => format!("Clean up {encoded} ({} versions)", stats.total_versions),
},
detail: match &pending_purge {
Some(token) => format!(
"purge requested at {token}: rewrite fragments with deletions \
and keep only the latest version"
),
None => format!(
"{} versions retained, above the {floor} floor; \
those outside the retention window can go",
stats.total_versions
),
},
summary: format!("Clean up {encoded} ({} versions)", stats.total_versions),
detail: format!(
"{} versions retained, above the {min_versions} floor; \
those outside the retention window can go",
stats.total_versions
),
parameters,
..Default::default()
});
@@ -312,33 +254,14 @@ impl JobHandler for CleanupVersionsHandler {
) as usize;
let client = self.client();
let mut table = dataset::open(&client, &id, &self.storage).await?;
let table = dataset::open(&client, &id, &self.storage).await?;
let before = table.stats().await?;
if let Some(token) = purge::pending(table.dataset.config()) {
return self
.execute_purge(request, sender, &job.job_id, &mut table, &token)
.await;
}
let Retention {
retain_hours,
min_versions,
} = Retention::for_table(
table.dataset.config(),
Retention {
retain_hours,
min_versions,
},
MAX_RETAIN_HOURS,
MAX_MIN_VERSIONS,
);
// The floor is a promise about how much history survives, so it has to
// be applied here and not only when the job was proposed: by the time it
// runs, versions may have aged past the retention window, and age alone
// would take the table below what the operator asked to keep.
let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await?
else {
let Some(floor) = version_floor(&table, min_versions).await? else {
sender.send_completed(JobCompleted {
request_id: request.request_id.clone(),
job_id: job.job_id.clone(),
@@ -365,6 +288,20 @@ impl JobHandler for CleanupVersionsHandler {
..Default::default()
})?;
let policy = CleanupPolicy {
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
// should_clean() ANDs its clauses, so a version has to be both older
// than the window and below the floor to go.
before_version: Some(floor),
// Files this dataset cannot account for are left alone: they may
// belong to a writer that has not committed yet, and deleting them
// would corrupt a commit in flight.
delete_unverified: false,
// A tagged version is pinned on purpose, so refuse rather than
// silently dropping what someone named.
error_if_tagged_old_versions: true,
..Default::default()
};
let stats = cleanup_old_versions(&table.dataset, policy)
.await
.with_context(|| format!("clean up versions of {}", table.location))?;
@@ -406,73 +343,3 @@ impl JobHandler for CleanupVersionsHandler {
Ok(())
}
}
impl CleanupVersionsHandler {
async fn execute_purge(
&self,
request: &ExecuteJobRequest,
sender: &dyn ExecutionSender,
job_id: &str,
table: &mut dataset::OpenTable,
token: &str,
) -> Result<()> {
let stats = purge::purge(&mut table.dataset, token)
.await
.with_context(|| format!("purge {}", table.location))?;
if let Some(counters) = &self.metrics {
counters.versions_removed.inc_by(stats.old_versions);
counters.bytes_reclaimed.inc_by(stats.bytes_removed);
}
let mut output: HashMap<String, ConfigValue> = HashMap::new();
output.insert(
"fragments_rewritten".to_string(),
int_value(stats.fragments_rewritten as i64),
);
output.insert(
"old_versions_removed".to_string(),
int_value(stats.old_versions as i64),
);
output.insert(
"bytes_removed".to_string(),
int_value(stats.bytes_removed as i64),
);
sender.send_completed(JobCompleted {
request_id: request.request_id.clone(),
job_id: job_id.to_string(),
job_type: JOB_TYPE.to_string(),
success: true,
result: Some(JobResult {
output_values: output,
summary: if stats.served {
format!(
"purged: rewrote {} fragments, removed {} versions and {} bytes",
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
)
} else {
format!(
"purge pending: rewrote {} fragments, removed {} versions and {} bytes; \
versions from before the rebuild are still retained",
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
)
},
..Default::default()
}),
..Default::default()
})?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_purge_gets_a_builds_execution_timeout() {
let descriptor = CleanupVersionsHandler::new(String::new()).descriptor();
let defaults = descriptor.admin_runtime_defaults.unwrap();
assert_eq!(defaults.execution_timeout_seconds, 3600);
assert_eq!(defaults.job_type_max_runtime_seconds, 3600);
assert!(!defaults.enabled);
}
}
@@ -1,757 +0,0 @@
use std::collections::HashMap;
use anyhow::{Result, anyhow, bail};
use async_trait::async_trait;
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field, string_value};
use seaweed_worker_core::pb::{
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
RunDetectionRequest, config_value::Kind,
};
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
use tracing::warn;
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::vector_index::{
EXECUTION_TIMEOUT_SECONDS, declare_build_runtime, is_vector_table,
};
use crate::jobs::{clamp, string_list, table_id};
use crate::metadata_index::{self, DEFAULT_MIN_UNINDEXED_ROWS, DEFAULT_SEGMENT_ROWS, State};
pub const JOB_TYPE: &str = "lance_metadata_index";
const SEGMENT_ROWS_FLOOR: i64 = 1_000;
const SEGMENT_ROWS_CEILING: i64 = 1_000_000_000;
const MIN_UNINDEXED_FLOOR: i64 = 0;
const MIN_UNINDEXED_CEILING: i64 = 100_000_000;
/// What a job does to one key.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Action {
/// Add the requested key's column, or finish a `building` one.
Add,
/// Drop an `off` key's index and column.
Remove,
/// Fill NULL rows and fold unindexed fragments of a `ready` key.
Maintain,
}
impl Action {
pub fn as_str(self) -> &'static str {
match self {
Action::Add => "add",
Action::Remove => "remove",
Action::Maintain => "maintain",
}
}
pub fn parse(s: &str) -> Option<Self> {
match s {
"add" => Some(Action::Add),
"remove" => Some(Action::Remove),
"maintain" => Some(Action::Maintain),
_ => None,
}
}
}
/// The work an index's table asks for: requested and `building` keys to add,
/// `off` keys to remove, and `ready` keys whose unindexed rows reached
/// `min_unindexed_rows`.
pub async fn pending(
dataset: &lance::Dataset,
min_unindexed_rows: u64,
) -> Result<Vec<(String, Action)>> {
let mut out: Vec<(String, Action)> = metadata_index::requests(dataset)
.into_iter()
.map(|key| (key, Action::Add))
.collect();
for key in metadata_index::keys_of(dataset) {
let action = match key.state {
Some(State::Building) => Action::Add,
Some(State::Off) => Action::Remove,
Some(State::Ready) => {
let segments = metadata_index::segments(dataset, &key.column).await?;
let pending = metadata_index::unindexed(dataset, &segments);
let unindexed: u64 = pending.iter().map(|(_, rows)| rows).sum();
if unindexed == 0 {
continue;
}
// Rows a write left NULL (one that raced the key's add and
// stopped before fixing them) wait for no threshold: queries
// through the key miss them until they are derived again.
// Nor does the first segment of a key that went ready on an
// empty table.
if unindexed < min_unindexed_rows
&& !segments.is_empty()
&& !has_nulls(dataset, &key.column, &pending).await?
{
continue;
}
Action::Maintain
}
None => continue,
};
out.push((key.key, action));
}
Ok(out)
}
/// Whether `fragments` hold a row whose `column` is NULL.
async fn has_nulls(
dataset: &lance::Dataset,
column: &str,
fragments: &[(u32, u64)],
) -> Result<bool> {
let ids: Vec<u64> = fragments.iter().map(|(id, _)| *id as u64).collect();
let mut scanner = dataset.scan();
scanner
.filter(&format!("`{}` IS NULL", column.replace('`', "``")))?
.with_fragments(
dataset
.fragments()
.iter()
.filter(|f| ids.contains(&f.id))
.cloned()
.collect(),
);
Ok(scanner.count_rows().await? > 0)
}
/// Carries out `action` on `key`, as the table now records it; an action the
/// table no longer asks for is skipped. Returns what was done.
pub async fn run(
dataset: &mut lance::Dataset,
key: &str,
action: Action,
segment_rows: u64,
min_unindexed_rows: u64,
grace: std::time::Duration,
) -> Result<String> {
let column = metadata_index::keys_of(dataset)
.into_iter()
.find(|k| k.key == key);
match (action, column) {
(Action::Add, None) => {
if !metadata_index::requests(dataset).iter().any(|k| k == key) {
return Ok(format!("{key} is no longer requested"));
}
let added = metadata_index::add_column(dataset, key).await?;
if added.state != Some(State::Building) {
return Ok(format!(
"{key} was withdrawn while {} was added",
added.column
));
}
let filled = metadata_index::finish_after(dataset, &added, segment_rows, grace).await?;
Ok(format!(
"{key} indexed in {} ({filled} rows a racing write left derived again)",
added.column
))
}
(Action::Add, Some(column)) if column.state == Some(State::Building) => {
// An add that stopped before setting again what its commit
// overwrote leaves that to the build resuming it.
let state = metadata_index::recover_add(dataset, &column).await?;
if state != Some(State::Building) {
return Ok(format!(
"{key} is {} after its add was recovered",
state.map_or("unknown", State::as_str)
));
}
let filled =
metadata_index::finish_after(dataset, &column, segment_rows, grace).await?;
Ok(format!(
"{key} indexed in {} ({filled} rows a racing write left derived again)",
column.column
))
}
(Action::Remove, Some(column)) if column.state == Some(State::Off) => {
// Dropping the column drops its restore mark: set again first
// what its add overwrote.
metadata_index::recover_add(dataset, &column).await?;
metadata_index::remove_column(dataset, &column.column).await?;
Ok(format!("{key} removed with {}", column.column))
}
(Action::Maintain, Some(column)) if column.state == Some(State::Ready) => {
let (filled, folded) =
metadata_index::maintain(dataset, &column, segment_rows, min_unindexed_rows)
.await?;
Ok(format!("{key}: {folded:?}, {filled} rows derived again"))
}
(action, column) => Ok(format!(
"{key} needs no {}: {}",
action.as_str(),
column
.and_then(|c| c.state)
.map_or("not indexed", State::as_str)
)),
}
}
/// Adds, maintains and removes the indexed metadata keys of S3 Vectors
/// indexes, as each index's table records them. The admin dispatches a job
/// when an operator asks; detection finds the rest.
pub struct MetadataIndexHandler {
namespace_url: String,
storage: dataset::Storage,
signing_key: Option<SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
}
impl MetadataIndexHandler {
pub fn new(namespace_url: String) -> Self {
Self {
namespace_url,
storage: dataset::Storage::default(),
signing_key: None,
metrics: None,
}
}
pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
self.metrics = metrics;
self
}
pub fn with_storage(mut self, storage: dataset::Storage) -> Self {
self.storage = storage;
self
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
fn client(&self) -> NamespaceClient {
NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref())
}
}
fn settings(values: &HashMap<String, ConfigValue>) -> (u64, u64) {
(
clamp(
int_or(values, "segment_rows", DEFAULT_SEGMENT_ROWS as i64),
SEGMENT_ROWS_FLOOR,
SEGMENT_ROWS_CEILING,
) as u64,
clamp(
int_or(
values,
"min_unindexed_rows",
DEFAULT_MIN_UNINDEXED_ROWS as i64,
),
MIN_UNINDEXED_FLOOR,
MIN_UNINDEXED_CEILING,
) as u64,
)
}
fn string_param(parameters: &HashMap<String, ConfigValue>, name: &str) -> Option<String> {
match parameters.get(name)?.kind.as_ref()? {
Kind::StringValue(s) => Some(s.clone()),
_ => None,
}
}
#[async_trait]
impl JobHandler for MetadataIndexHandler {
fn capability(&self) -> JobTypeCapability {
JobTypeCapability {
job_type: JOB_TYPE.to_string(),
can_detect: true,
can_execute: true,
max_detection_concurrency: 1,
max_execution_concurrency: 1,
display_name: "Lance Metadata Index".to_string(),
description: "Add, maintain and remove the indexed metadata keys of S3 Vectors indexes"
.to_string(),
weight: 40,
}
}
fn descriptor(&self) -> JobTypeDescriptor {
let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
defaults.insert(
"segment_rows".to_string(),
int_value(DEFAULT_SEGMENT_ROWS as i64),
);
defaults.insert(
"min_unindexed_rows".to_string(),
int_value(DEFAULT_MIN_UNINDEXED_ROWS as i64),
);
JobTypeDescriptor {
job_type: JOB_TYPE.to_string(),
display_name: "Lance Metadata Index".to_string(),
description:
"Keep a LABEL_LIST index for each metadata key an S3 Vectors index asks for"
.to_string(),
icon: "fas fa-tags".to_string(),
descriptor_version: 1,
worker_config_form: Some(form(
"lance-metadata-index-worker",
"Metadata index",
vec![
number_field(
"segment_rows",
"Segment rows",
"Rows an index segment takes before a new one starts",
SEGMENT_ROWS_FLOOR,
SEGMENT_ROWS_CEILING,
),
number_field(
"min_unindexed_rows",
"Unindexed rows",
"Rows written since the last fold that wait for the next one",
MIN_UNINDEXED_FLOOR,
MIN_UNINDEXED_CEILING,
),
],
defaults.clone(),
)),
worker_default_values: defaults,
admin_runtime_defaults: Some(AdminRuntimeDefaults {
job_type_max_runtime_seconds: EXECUTION_TIMEOUT_SECONDS,
execution_timeout_seconds: EXECUTION_TIMEOUT_SECONDS,
..Default::default()
}),
..Default::default()
}
}
async fn detect(
&self,
request: &RunDetectionRequest,
sender: &dyn DetectionSender,
) -> Result<()> {
let (_, min_unindexed_rows) = settings(&request.worker_config_values);
let max_results = usize::try_from(request.max_results).unwrap_or(0);
let client = self.client();
let mut proposals = Vec::new();
let mut has_more = false;
'tables: for encoded in &client.list_all_tables().await? {
let id = parse_id(encoded);
if !is_vector_table(&id) {
continue;
}
let table = match dataset::open(&client, &id, &self.storage).await {
Ok(table) => table,
Err(err) => {
if let Some(counters) = &self.metrics {
counters.worker.object_skipped(JOB_TYPE, "open");
}
warn!("skipping {encoded}: {err:#}");
continue;
}
};
if let Some(counters) = &self.metrics {
counters.worker.object_seen(JOB_TYPE);
}
let work = match pending(&table.dataset, min_unindexed_rows).await {
Ok(work) => work,
Err(err) => {
warn!("skipping {encoded}: reading its indexed keys failed: {err:#}");
continue;
}
};
for (key, action) in work {
if max_results > 0 && proposals.len() >= max_results {
has_more = true;
break 'tables;
}
proposals.push(proposal(&id, encoded, &key, action));
}
}
let total = proposals.len() as i32;
sender.send_proposals(DetectionProposals {
request_id: request.request_id.clone(),
job_type: JOB_TYPE.to_string(),
proposals,
has_more,
})?;
sender.send_complete(DetectionComplete {
request_id: request.request_id.clone(),
job_type: JOB_TYPE.to_string(),
success: true,
error_message: String::new(),
total_proposals: total,
skipped_reason: String::new(),
eligible_after: None,
})?;
Ok(())
}
async fn execute(
&self,
request: &ExecuteJobRequest,
sender: &dyn ExecutionSender,
) -> Result<()> {
let job = request
.job
.as_ref()
.ok_or_else(|| anyhow!("execute request carried no job"))?;
let id = table_id(&job.parameters)
.ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
if !is_vector_table(&id) {
bail!("{} is not an S3 Vectors index", id.join("/"));
}
let key = string_param(&job.parameters, "key")
.ok_or_else(|| anyhow!("job {} carried no key", job.job_id))?;
let action = string_param(&job.parameters, "action")
.as_deref()
.and_then(Action::parse)
.ok_or_else(|| anyhow!("job {} carried no valid action", job.job_id))?;
let (segment_rows, min_unindexed_rows) = settings(&request.worker_config_values);
sender.send_progress(JobProgressUpdate {
request_id: request.request_id.clone(),
job_id: job.job_id.clone(),
job_type: JOB_TYPE.to_string(),
progress_percent: 10.0,
stage: format!("{} {key}", action.as_str()),
..Default::default()
})?;
let client = self.client();
let mut table = dataset::open(&client, &id, &self.storage).await?;
let summary = run(
&mut table.dataset,
&key,
action,
segment_rows,
min_unindexed_rows,
metadata_index::RACE_GRACE,
)
.await?;
sender.send_completed(JobCompleted {
request_id: request.request_id.clone(),
job_id: job.job_id.clone(),
job_type: JOB_TYPE.to_string(),
success: true,
result: Some(JobResult {
summary,
..Default::default()
}),
..Default::default()
})?;
Ok(())
}
}
/// The proposal for one key of one table; the admin builds the same
/// parameters when it dispatches a job for an operator.
fn proposal(id: &[String], encoded: &str, key: &str, action: Action) -> JobProposal {
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
parameters.insert("table_id".to_string(), string_list(id));
parameters.insert("key".to_string(), string_value(key));
parameters.insert("action".to_string(), string_value(action.as_str()));
declare_build_runtime(&mut parameters);
let dedupe = format!("{JOB_TYPE}:{encoded}:{key}:{}", action.as_str());
JobProposal {
proposal_id: dedupe.clone(),
dedupe_key: dedupe,
job_type: JOB_TYPE.to_string(),
summary: format!("{} metadata key {key} of {encoded}", action.as_str()),
parameters,
..Default::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_proposal_names_the_table_key_and_action() {
let id = parse_id("media$vector$movies");
let p = proposal(&id, "media$vector$movies", "author", Action::Remove);
assert_eq!(table_id(&p.parameters), Some(id));
assert_eq!(
string_param(&p.parameters, "key").as_deref(),
Some("author")
);
assert_eq!(
string_param(&p.parameters, "action")
.as_deref()
.and_then(Action::parse),
Some(Action::Remove)
);
assert_eq!(
p.dedupe_key,
"lance_metadata_index:media$vector$movies:author:remove"
);
}
#[test]
fn settings_are_held_to_their_ranges() {
let mut values = HashMap::new();
assert_eq!(
settings(&values),
(DEFAULT_SEGMENT_ROWS, DEFAULT_MIN_UNINDEXED_ROWS)
);
values.insert("segment_rows".to_string(), int_value(-5));
values.insert("min_unindexed_rows".to_string(), int_value(-1));
assert_eq!(settings(&values), (SEGMENT_ROWS_FLOOR as u64, 0));
}
/// A removal of a key whose add stopped before setting again what it
/// overwrote sets that first: a key requested meanwhile stays requested.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_removal_keeps_what_the_add_overwrote() {
use crate::metadata_index::tests::lifecycle::table;
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 2).await;
metadata_index::request(&mut dataset, "author")
.await
.unwrap();
let mut adder = dataset.clone();
metadata_index::request(&mut dataset, "lang").await.unwrap();
metadata_index::commit_column(&mut adder, "author")
.await
.unwrap();
metadata_index::request_removal(&mut adder, "author")
.await
.unwrap();
run(
&mut adder,
"author",
Action::Remove,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert!(metadata_index::keys_of(&adder).is_empty());
assert_eq!(metadata_index::requests(&adder), ["lang"]);
}
/// A withdrawal of a request an interrupted add overwrote sets that
/// first, so recovering the add does not bring the request back.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withdrawal_of_a_request_an_add_overwrote_stays_withdrawn() {
use crate::metadata_index::tests::lifecycle::table;
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 2).await;
metadata_index::request(&mut dataset, "author")
.await
.unwrap();
let mut adder = dataset.clone();
metadata_index::request(&mut dataset, "lang").await.unwrap();
metadata_index::commit_column(&mut adder, "author")
.await
.unwrap();
dataset.checkout_latest().await.unwrap();
metadata_index::request_removal(&mut dataset, "lang")
.await
.unwrap();
let author = metadata_index::keys_of(&dataset).remove(0);
metadata_index::recover_add(&mut dataset, &author)
.await
.unwrap();
assert!(metadata_index::requests(&dataset).is_empty());
}
/// A withdrawal that read the request before the key's column was added
/// and committed after sets the column off.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withdrawal_racing_the_add_sets_the_column_off() {
use crate::metadata_index::tests::lifecycle::table;
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 2).await;
metadata_index::request(&mut dataset, "genre")
.await
.unwrap();
let mut operator = dataset.clone();
let added = metadata_index::add_column(&mut dataset, "genre")
.await
.unwrap();
assert_eq!(added.state, Some(State::Building));
metadata_index::request_removal(&mut operator, "genre")
.await
.unwrap();
dataset.checkout_latest().await.unwrap();
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
assert!(metadata_index::requests(&dataset).is_empty());
}
/// A key that went ready on an empty table builds its first segment
/// once rows arrive, below the threshold.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_key_ready_without_a_segment_is_maintained_at_once() {
use crate::metadata_index::tests::lifecycle::{table, upsert};
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 1).await;
dataset.delete("true").await.unwrap();
metadata_index::request(&mut dataset, "author")
.await
.unwrap();
run(
&mut dataset,
"author",
Action::Add,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert_eq!(
metadata_index::keys_of(&dataset)[0].state,
Some(State::Ready)
);
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
let mut dataset = upsert(dataset, 0..100).await;
assert_eq!(
pending(&dataset, 1_000_000).await.unwrap(),
[("author".to_string(), Action::Maintain)]
);
run(
&mut dataset,
"author",
Action::Maintain,
1_000,
1_000_000,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert_eq!(
metadata_index::segments(&dataset, "_mk_0")
.await
.unwrap()
.len(),
1
);
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
}
/// What the table asks for is what a job does, and a job the table no
/// longer asks for does nothing.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn jobs_follow_what_the_table_records() {
use crate::metadata_index::tests::lifecycle::{table, upsert};
let dir = tempfile::TempDir::new().unwrap();
let mut dataset = table(&dir, 4).await;
assert!(pending(&dataset, 0).await.unwrap().is_empty());
metadata_index::request(&mut dataset, "author")
.await
.unwrap();
metadata_index::request(&mut dataset, "genre")
.await
.unwrap();
assert_eq!(
pending(&dataset, 0).await.unwrap(),
[
("author".to_string(), Action::Add),
("genre".to_string(), Action::Add)
]
);
metadata_index::request_removal(&mut dataset, "genre")
.await
.unwrap();
let skipped = run(
&mut dataset,
"genre",
Action::Add,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert_eq!(skipped, "genre is no longer requested");
run(
&mut dataset,
"author",
Action::Add,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
let keys = metadata_index::keys_of(&dataset);
assert_eq!(keys.len(), 1);
assert_eq!(keys[0].state, Some(State::Ready));
assert_eq!(
metadata_index::segments(&dataset, "_mk_0")
.await
.unwrap()
.len(),
4
);
assert!(pending(&dataset, 0).await.unwrap().is_empty());
let mut dataset = upsert(dataset, 4_000..4_500).await;
assert!(pending(&dataset, 1_000).await.unwrap().is_empty());
assert_eq!(
pending(&dataset, 500).await.unwrap(),
[("author".to_string(), Action::Maintain)]
);
run(
&mut dataset,
"author",
Action::Maintain,
1_000,
500,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert!(pending(&dataset, 0).await.unwrap().is_empty());
// A raced write that left NULL rows is maintained at once.
let stale = dataset.clone();
let mut dataset =
crate::metadata_index::tests::lifecycle::upsert_with_null_columns(stale, 4_500..4_520)
.await;
assert_eq!(
pending(&dataset, 1_000_000).await.unwrap(),
[("author".to_string(), Action::Maintain)]
);
run(
&mut dataset,
"author",
Action::Maintain,
1_000,
1_000_000,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
let noop = run(
&mut dataset,
"author",
Action::Remove,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert_eq!(noop, "author needs no remove: ready");
metadata_index::request_removal(&mut dataset, "author")
.await
.unwrap();
assert_eq!(
pending(&dataset, 0).await.unwrap(),
[("author".to_string(), Action::Remove)]
);
run(
&mut dataset,
"author",
Action::Remove,
1_000,
0,
std::time::Duration::ZERO,
)
.await
.unwrap();
assert!(metadata_index::keys_of(&dataset).is_empty());
assert!(pending(&dataset, 0).await.unwrap().is_empty());
}
}
@@ -9,8 +9,6 @@
pub mod cleanup;
pub mod compact;
pub mod indices;
pub mod metadata_index;
pub mod purge;
pub mod sort;
pub mod vector_index;
@@ -67,12 +65,6 @@ pub fn handlers(
.with_signing_key(signing_key.clone())
.with_metrics(metrics.clone()),
),
Arc::new(
metadata_index::MetadataIndexHandler::new(namespace_url.clone())
.with_storage(storage.clone())
.with_signing_key(signing_key.clone())
.with_metrics(metrics.clone()),
),
Arc::new(
sort::SortHandler::new(namespace_url.clone())
.with_storage(storage.clone())
@@ -1,328 +0,0 @@
//! Per-table retention for version cleanup, and purges: compaction that
//! rewrites every fragment with deletions followed by a cleanup that keeps
//! only the latest version, so the bytes of deleted rows leave storage now
//! rather than after the retention window.
//!
//! Both are set in the table's own Lance config, so any client that can write
//! the table can set them and they travel with it:
//!
//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions`
//! override the worker's retention for this table.
//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`]
//! writes the current time); a purge is pending while it differs from
//! `seaweedfs.purge.completed`, which the purge sets to the token it served.
//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use anyhow::{Context, Result, bail};
use chrono::Utc;
use lance::Dataset;
use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions};
use lance::dataset::optimize::{CompactionOptions, compact_files};
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
use lance_index::metrics::NoOpMetricsCollector;
use lance_index::optimize::OptimizeOptions;
use lance_index::{IndexType, is_system_index};
use lance_table::format::IndexMetadata;
use tracing::warn;
use crate::jobs::cleanup::retention_policy;
pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours";
pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions";
pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested";
pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed";
pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt";
/// The retention cleanup applies to one table: the worker's, unless the
/// table's config overrides a value with one inside the worker's range.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Retention {
pub retain_hours: i64,
pub min_versions: usize,
}
impl Retention {
pub fn for_table(
config: &HashMap<String, String>,
worker: Retention,
max_retain_hours: i64,
max_min_versions: i64,
) -> Retention {
let read = |key: &str, min: i64, max: i64| -> Option<i64> {
let raw = config.get(key)?;
match raw.trim().parse::<i64>() {
Ok(value) if (min..=max).contains(&value) => Some(value),
_ => {
warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}");
None
}
}
};
Retention {
retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours)
.unwrap_or(worker.retain_hours),
min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions)
.map(|v| v as usize)
.unwrap_or(worker.min_versions),
}
}
}
/// The request token of a purge the table is waiting for, if any.
pub fn pending(config: &HashMap<String, String>) -> Option<String> {
let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?;
if requested.is_empty() {
return None;
}
match config.get(PURGE_COMPLETED_KEY) {
Some(completed) if completed.trim() == requested => None,
_ => Some(requested.to_string()),
}
}
/// Records a purge request on the table and returns its token. The next
/// cleanup detection proposes the purge.
pub async fn request_purge(dataset: &mut Dataset) -> Result<String> {
let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
dataset
.update_config([(PURGE_REQUESTED_KEY, token.as_str())])
.await
.context("record purge request")?;
Ok(token)
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct PurgeStats {
pub fragments_rewritten: usize,
pub old_versions: u64,
pub bytes_removed: u64,
/// False while versions from before the rebuild remain.
pub served: bool,
}
/// Rewrites every fragment that has deleted rows and rebuilds the indices from
/// the rows that remain, then removes the versions from before that through
/// the cleanup job's retention policy with only the latest version kept, and
/// marks `token` served once none of them is left. The rebuild is needed
/// because compaction only remaps an index: an IVF_FLAT index keeps the
/// deleted vector's values in its partitions under a remapped row address. It
/// runs once per request, so a purge that has to wait for its cleanup (a
/// version pinned by a tag, a query lease or one someone named, fails it)
/// retries only the cleanup. Files no version references stay while Lance
/// cannot tell them from a writer's uncommitted ones, which is up to seven
/// days for an abandoned write.
pub async fn purge(dataset: &mut Dataset, token: &str) -> Result<PurgeStats> {
let fragments_rewritten = rebuild(dataset, token).await?;
let cleanup = match retention_policy(dataset, 0, 1).await? {
Some(policy) => cleanup_old_versions(dataset, policy)
.await
.context("remove versions older than the purge")?,
None => RemovalStats::default(),
};
let served = finish(dataset, token).await?;
Ok(PurgeStats {
fragments_rewritten,
old_versions: cleanup.old_versions,
bytes_removed: cleanup.bytes_removed,
served,
})
}
/// Rewrites the fragments with deletions and rebuilds the indices for
/// `token`, unless that was done already, and returns how many fragments it
/// rewrote.
pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result<usize> {
if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) {
return Ok(0);
}
let vector_indices = vector_index_names(dataset).await?;
let options = CompactionOptions {
materialize_deletions: true,
// Zero materializes every fragment with any deletion.
materialize_deletions_threshold: 0.0,
..Default::default()
};
let compaction = compact_files(dataset, options, None)
.await
.context("rewrite fragments with deletions")?;
dataset.checkout_latest().await?;
if !vector_indices.is_empty() {
let mut options = OptimizeOptions::retrain();
options.index_names = Some(vector_indices);
dataset
.optimize_indices(&options)
.await
.context("rebuild indices without the deleted rows")?;
}
rebuild_scalar_indices(dataset).await?;
dataset.checkout_latest().await?;
dataset
.update_config([(PURGE_REBUILT_KEY, token)])
.await
.context("record purge rebuild")?;
Ok(compaction.fragments_removed)
}
/// Marks `token` served when the oldest version left is one the rebuild for
/// it wrote, so no version from before the rebuild remains.
pub async fn finish(dataset: &mut Dataset, token: &str) -> Result<bool> {
let oldest = dataset
.versions()
.await?
.iter()
.map(|v| v.version)
.min()
.context("the table has no versions")?;
let oldest = dataset.checkout_version(oldest).await?;
if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) {
return Ok(false);
}
dataset
.update_config([(PURGE_COMPLETED_KEY, token)])
.await
.context("record purge completion")?;
Ok(true)
}
/// Names of the vector indices. A retrain builds each name over the whole
/// table, so a column whose index is sharded over several names is refused:
/// its shards would overlap.
async fn vector_index_names(dataset: &Dataset) -> Result<Vec<String>> {
let mut by_column: BTreeMap<Vec<i32>, BTreeSet<String>> = BTreeMap::new();
for index in dataset
.load_indices()
.await?
.iter()
.filter(|i| is_vector(i))
{
by_column
.entry(index.fields.clone())
.or_default()
.insert(index.name.clone());
}
let mut names = Vec::new();
for shards in by_column.into_values() {
if shards.len() > 1 {
bail!(
"the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table"
);
}
names.extend(shards);
}
Ok(names)
}
/// Builds every scalar index again from the rows that remain. Compaction only
/// remaps them: a bitmap index keeps a deleted key, and under stable row ids
/// no scalar index is rewritten at all.
async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> {
let indices = dataset.load_indices().await?;
let mut rebuilt = BTreeSet::new();
for index in indices.iter() {
if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) {
continue;
}
let [field] = index.fields[..] else {
bail!("index {} covers {} columns", index.name, index.fields.len());
};
let column = dataset.schema().field_path(field)?;
let params = dataset
.open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector)
.await?
.derive_index_params()?;
dataset
.create_index(
&[column.as_str()],
IndexType::Scalar,
Some(index.name.clone()),
&params,
true,
)
.await
.with_context(|| format!("rebuild index {}", index.name))?;
}
Ok(())
}
fn is_vector(index: &IndexMetadata) -> bool {
index
.index_details
.as_ref()
.is_some_and(|details| details.type_url.ends_with("VectorIndexDetails"))
}
#[cfg(test)]
mod tests {
use super::*;
fn config(pairs: &[(&str, &str)]) -> HashMap<String, String> {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
const WORKER: Retention = Retention {
retain_hours: 168,
min_versions: 5,
};
#[test]
fn table_retention_overrides_within_range() {
assert_eq!(
Retention::for_table(&config(&[]), WORKER, 8760, 1000),
WORKER
);
assert_eq!(
Retention::for_table(
&config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]),
WORKER,
8760,
1000
),
Retention {
retain_hours: 0,
min_versions: 1
}
);
for bad in ["-1", "8761", "x", ""] {
assert_eq!(
Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000)
.retain_hours,
168,
"{bad:?}"
);
}
assert_eq!(
Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000)
.min_versions,
5
);
}
#[test]
fn purge_is_pending_until_its_token_is_served() {
assert_eq!(pending(&config(&[])), None);
assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None);
assert_eq!(
pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])),
Some("t1".to_string())
);
assert_eq!(
pending(&config(&[
(PURGE_REQUESTED_KEY, "t2"),
(PURGE_COMPLETED_KEY, "t1")
])),
Some("t2".to_string())
);
assert_eq!(
pending(&config(&[
(PURGE_REQUESTED_KEY, "t2"),
(PURGE_COMPLETED_KEY, "t2")
])),
None
);
}
}
Loaded 100 of 616 files, more files were not shown because too many files have changed in this diff. Show more