mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 19:57:39 +01:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f400a0c696 | ||
|
|
a957802c6d | ||
|
|
19b51cec32 | ||
|
|
916bba2609 | ||
|
|
8b783a3933 | ||
|
|
fb3a41f224 | ||
|
|
f1eac773cc | ||
|
|
15e7aecdaa | ||
|
|
bf02a57bf2 | ||
|
|
f697808439 | ||
|
|
8ce51ce629 | ||
|
|
7144c3c5cf | ||
|
|
1fb0d8d53f |
No files matched your search
@@ -22,16 +22,6 @@ CHART = ROOT / "k8s/charts/seaweedfs"
|
||||
SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
|
||||
KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
|
||||
ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
|
||||
# No workload is needed to exercise Helm's real lookup and update.
|
||||
RELEASE_SETTINGS = (
|
||||
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
|
||||
"global.seaweedfs.createClusterRole=false",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
|
||||
)
|
||||
CONFIGMAP_CHART = ("seaweedfs", "--repo", "https://seaweedfs.github.io/seaweedfs/helm", "--version", "4.54.0")
|
||||
|
||||
|
||||
def run(*args):
|
||||
@@ -87,7 +77,8 @@ def fixtures():
|
||||
|
||||
|
||||
def check_generated(value):
|
||||
assert len(value) == 64 and value.isascii() and value.isalnum(), "invalid generated JWT key"
|
||||
decoded = base64.b64decode(value, validate=True).decode("ascii")
|
||||
assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
|
||||
|
||||
|
||||
def check_helpers(helm, reference_helm):
|
||||
@@ -156,11 +147,20 @@ def check_upgrades(helm, context):
|
||||
legacy = "seaweedfs-security-config"
|
||||
kubectl = ["kubectl", "--context", context, "-n", namespace]
|
||||
release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
|
||||
for setting in RELEASE_SETTINGS:
|
||||
# No workload is needed to exercise Helm's real ConfigMap lookup and update.
|
||||
for setting in (
|
||||
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
|
||||
"global.seaweedfs.createClusterRole=false",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
|
||||
):
|
||||
release_args += ["--set", setting]
|
||||
|
||||
def stored():
|
||||
return secret_keys(kubectl, current)
|
||||
cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
|
||||
return keys(cm["data"]["security.toml"])
|
||||
|
||||
def upgrade():
|
||||
run(helm, "upgrade", *release_args)
|
||||
@@ -168,8 +168,8 @@ def check_upgrades(helm, context):
|
||||
|
||||
def patch(raw):
|
||||
# Seed previous-release content without taking Helm 4's SSA ownership.
|
||||
run(*kubectl, "patch", "secret", current, "--type=merge", "--field-manager=helm", "-p",
|
||||
json.dumps({"stringData": {"security.toml": raw}}))
|
||||
run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
|
||||
json.dumps({"data": {"security.toml": raw}}))
|
||||
|
||||
run(*kubectl, "create", "namespace", namespace)
|
||||
try:
|
||||
@@ -194,24 +194,17 @@ def check_upgrades(helm, context):
|
||||
assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
|
||||
print(f"PASS upgrade: {name}")
|
||||
|
||||
# Migration from the ConfigMaps of earlier chart versions, then precedence of the Secret.
|
||||
configmap_raw = "\n".join(f'[{section}]\nkey = "configmap-{index}"'
|
||||
for index, section in enumerate(SECTIONS))
|
||||
run(*kubectl, "create", "configmap", current, "--from-literal=security.toml=" + configmap_raw)
|
||||
run(*kubectl, "delete", "secret", current)
|
||||
assert upgrade() == keys(configmap_raw), "ConfigMap keys were not preserved"
|
||||
print("PASS upgrade: ConfigMap migration")
|
||||
# Migration from the old chart name, followed by precedence of the current name.
|
||||
legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
|
||||
for index, section in enumerate(SECTIONS))
|
||||
run(*kubectl, "delete", "configmap", current)
|
||||
run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
|
||||
run(*kubectl, "delete", "secret", current)
|
||||
run(*kubectl, "delete", "configmap", current)
|
||||
assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
|
||||
print("PASS upgrade: legacy ConfigMap migration")
|
||||
current_raw = next(fixtures())[1]
|
||||
patch(current_raw)
|
||||
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode the Secret"
|
||||
print("PASS upgrade: Secret takes precedence")
|
||||
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
|
||||
print("PASS upgrade: current ConfigMap takes precedence")
|
||||
|
||||
patch(ESCAPED_HEADER)
|
||||
try:
|
||||
@@ -220,76 +213,13 @@ def check_upgrades(helm, context):
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
assert stored()["jwt.signing"] == "existing"
|
||||
assert keys(run(*kubectl, "get", "configmap", current, "-o",
|
||||
"jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
|
||||
print("PASS upgrade: unsupported quoted header leaves stored key untouched")
|
||||
finally:
|
||||
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
|
||||
|
||||
|
||||
def secret_keys(kubectl, name):
|
||||
secret = json.loads(run(*kubectl, "get", "secret", name, "-o", "json"))
|
||||
return keys(base64.b64decode(secret["data"]["security.toml"]).decode())
|
||||
|
||||
|
||||
def check_configmap_release_upgrade(helm, context):
|
||||
namespace = "jwt-key-migrate-" + uuid.uuid4().hex[:8]
|
||||
name = "jk-seaweedfs-security-config"
|
||||
kubectl = ["kubectl", "--context", context, "-n", namespace]
|
||||
common = ["--kube-context", context, "-n", namespace]
|
||||
for setting in RELEASE_SETTINGS:
|
||||
common += ["--set", setting]
|
||||
upgrade = ["upgrade", "jk", str(CHART), *common, "--set", "master.enabled=true"]
|
||||
|
||||
def refuse_partitioned(stage):
|
||||
try:
|
||||
run(helm, *upgrade, "--set", "master.updatePartition=1")
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "updatePartition=0" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError(f"partitioned upgrade removed the ConfigMap ({stage})")
|
||||
|
||||
def exists(kind):
|
||||
return run(*kubectl, "get", kind, name, "--ignore-not-found", "-o", "name").strip()
|
||||
|
||||
run(*kubectl, "create", "namespace", namespace)
|
||||
try:
|
||||
run(helm, "install", "jk", *CONFIGMAP_CHART, *common)
|
||||
before = keys(run(*kubectl, "get", "configmap", name, "-o", "jsonpath={.data.security\\.toml}"))
|
||||
assert set(before) == set(SECTIONS), "ConfigMap chart omitted a JWT section"
|
||||
refuse_partitioned("ConfigMap release")
|
||||
print("PASS upgrade: partitioned upgrade from the ConfigMap chart is refused")
|
||||
|
||||
# Fail an upgrade after Helm created the Secret, so it never removes the ConfigMap.
|
||||
run(*kubectl, "create", "quota", "no-statefulsets", "--hard=count/statefulsets.apps=0")
|
||||
try:
|
||||
run(helm, *upgrade)
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "quota" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("upgrade was not blocked by the quota")
|
||||
run(*kubectl, "delete", "quota", "no-statefulsets")
|
||||
assert exists("secret") and exists("configmap"), "failed upgrade did not leave both objects"
|
||||
refuse_partitioned("retry after a failed upgrade")
|
||||
print("PASS upgrade: partitioned retry after a failed migration is refused")
|
||||
|
||||
run(helm, "upgrade", "jk", str(CHART), *common)
|
||||
assert secret_keys(kubectl, name) == before, "upgrade from the ConfigMap chart changed a key"
|
||||
assert not exists("configmap"), "upgrade left the ConfigMap behind"
|
||||
print("PASS upgrade: ConfigMap chart release moves its keys to the Secret")
|
||||
|
||||
# The unprefixed legacy name can belong to another release in the namespace.
|
||||
other = "seaweedfs-security-config"
|
||||
run(*kubectl, "create", "configmap", other, '--from-literal=security.toml=[jwt.signing]\nkey = "other"')
|
||||
run(*kubectl, "annotate", "configmap", other, "meta.helm.sh/release-name=other",
|
||||
"meta.helm.sh/release-namespace=" + namespace)
|
||||
run(helm, *upgrade, "--set", "master.updatePartition=1")
|
||||
assert secret_keys(kubectl, name) == before, "another release's ConfigMap changed a key"
|
||||
assert run(*kubectl, "get", "configmap", other, "-o", "name").strip(), "removed another release's ConfigMap"
|
||||
print("PASS upgrade: another release's legacy ConfigMap does not block a partitioned upgrade")
|
||||
finally:
|
||||
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--helm", default="helm")
|
||||
@@ -300,7 +230,6 @@ def main():
|
||||
check_helpers(args.helm, args.reference_helm)
|
||||
if args.context:
|
||||
check_upgrades(args.helm, args.context)
|
||||
check_configmap_release_upgrade(args.helm, args.context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regroup a GitHub release's generated notes by component.
|
||||
|
||||
GitHub's generated notes are one flat list of PRs in merge order. This keeps every
|
||||
line as GitHub wrote it, but sorts the PRs under component headings (S3 API, Filer,
|
||||
Mount, Volume Server, ...), with dependency bumps folded at the end.
|
||||
|
||||
The component comes from the PR title prefix, which nearly every PR has
|
||||
("s3api: ...", "fix(mount): ...", "filer.sync: ...", "rust volume: ..."). A PR with
|
||||
no recognised prefix is placed by the files it changes.
|
||||
|
||||
Usage:
|
||||
release_notes_by_component.py --tag 4.54 # print the regrouped notes
|
||||
release_notes_by_component.py --tag 4.54 --write # also update the release
|
||||
|
||||
Running it again on notes it already regrouped gives the same result.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Display order. Each PR lands in exactly one section.
|
||||
SECTIONS = [
|
||||
"S3 API",
|
||||
"S3 Tables and Iceberg",
|
||||
"Vectors",
|
||||
"Filer",
|
||||
"Mount",
|
||||
"Volume Server",
|
||||
"Rust Volume Server",
|
||||
"Erasure Coding",
|
||||
"Master and Topology",
|
||||
"Admin UI and Workers",
|
||||
"Shell",
|
||||
"Security",
|
||||
"Message Queue",
|
||||
"Kubernetes, Helm and Deployment",
|
||||
"Core and CLI",
|
||||
"Documentation",
|
||||
"Tests and CI",
|
||||
"Other",
|
||||
"Dependencies",
|
||||
]
|
||||
|
||||
# Title scope -> section, first match wins. Matched against each lower-cased scope
|
||||
# token, e.g. "filer.sync", "s3 lifecycle", "weed/s3api" (a "weed/" prefix is
|
||||
# dropped first). Order matters: specific scopes come before the general ones
|
||||
# they start with (s3tables before s3, volume.fsck before volume).
|
||||
SCOPE_RULES = [
|
||||
(r"(build\(deps\)|deps|dependabot)\b", "Dependencies"),
|
||||
(r"(test|tests|e2e|integration)\b|.*\btests?$|test/", "Tests and CI"),
|
||||
(r"(rust[ -]?volume|seaweed-volume|volume\(rust\)|rust)\b", "Rust Volume Server"),
|
||||
(r"(s3 ?vectors?|vectors?|seaweed-vector|weed-vector|lance)\b", "Vectors"),
|
||||
(r"(s3 ?tables|iceberg)\b", "S3 Tables and Iceberg"),
|
||||
(r"(s3 sink|remote|remote_storage|filer|webdav|sftp|ftp|notification|replication|"
|
||||
r"redis\d*|leveldb\d*|backup)\b", "Filer"),
|
||||
(r"(s3|s3api|iam|oidc|sts|credential|sse|cors|policy|object ?lock)\b", "S3 API"),
|
||||
(r"(mount|fuse)\b", "Mount"),
|
||||
(r"(ec|ec_\w+|ec\.\w+|erasure[ _-]?coding|worker/ec)\b", "Erasure Coding"),
|
||||
(r"(master|topology|wdclient|cluster|raft|placement)\b", "Master and Topology"),
|
||||
(r"(admin|worker|weed-worker|plugin|plugin worker|maintenance)\b",
|
||||
"Admin UI and Workers"),
|
||||
(r"(shell|fsck|volume\.\w+|fs\.\w+|collection\.\w+|bucket\.\w+)", "Shell"),
|
||||
(r"(volume|volume_\w+|storage|vacuum|operation|compression|needle)\b", "Volume Server"),
|
||||
(r"(security|tls|mtls|jwt|grpc|kms)\b", "Security"),
|
||||
(r"(mq|kafka|messaging)\b", "Message Queue"),
|
||||
(r"(helm|k8s|kubernetes|operator|csi|docker|grafana|install\.sh|chart|snap|terraform)\b",
|
||||
"Kubernetes, Helm and Deployment"),
|
||||
(r"(docs?|readme|wiki)\b", "Documentation"),
|
||||
(r"(ci|release|build|github|workflow|actions)\b", "Tests and CI"),
|
||||
(r"(server|command|mini|weed-db|weed|util|pb|cmd|glog|stats|telemetry|query|config)\b",
|
||||
"Core and CLI"),
|
||||
]
|
||||
|
||||
# Changed path prefix -> section, for PRs whose title has no usable prefix.
|
||||
# Longest prefix wins.
|
||||
PATH_RULES = {
|
||||
"weed/s3api/s3tables/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/iceberg/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/s3vectors/": "Vectors",
|
||||
"weed/s3api/lance/": "Vectors",
|
||||
"seaweed-vector/": "Vectors",
|
||||
"weed/s3api/": "S3 API",
|
||||
"weed/iam/": "S3 API",
|
||||
"weed/iamapi/": "S3 API",
|
||||
"weed/credential/": "S3 API",
|
||||
"weed/filer/": "Filer",
|
||||
"weed/filer_client/": "Filer",
|
||||
"weed/server/filer": "Filer",
|
||||
"weed/server/webdav": "Filer",
|
||||
"weed/sftpd/": "Filer",
|
||||
"weed/replication/": "Filer",
|
||||
"weed/notification/": "Filer",
|
||||
"weed/remote_storage/": "Filer",
|
||||
"weed/mount/": "Mount",
|
||||
"weed/storage/erasure_coding/": "Erasure Coding",
|
||||
"weed/ec/": "Erasure Coding",
|
||||
"weed/worker/tasks/erasure_coding/": "Erasure Coding",
|
||||
"weed/storage/": "Volume Server",
|
||||
"weed/server/volume": "Volume Server",
|
||||
"weed/operation/": "Volume Server",
|
||||
"seaweed-volume/": "Rust Volume Server",
|
||||
"weed/topology/": "Master and Topology",
|
||||
"weed/server/master": "Master and Topology",
|
||||
"weed/wdclient/": "Master and Topology",
|
||||
"weed/cluster/": "Master and Topology",
|
||||
"weed/placement/": "Master and Topology",
|
||||
"weed/admin/": "Admin UI and Workers",
|
||||
"weed/worker/": "Admin UI and Workers",
|
||||
"weed/plugin/": "Admin UI and Workers",
|
||||
"seaweed-worker/": "Admin UI and Workers",
|
||||
"weed/shell/": "Shell",
|
||||
"weed/security/": "Security",
|
||||
"weed/kms/": "Security",
|
||||
"weed/mq/": "Message Queue",
|
||||
"k8s/": "Kubernetes, Helm and Deployment",
|
||||
"docker/": "Kubernetes, Helm and Deployment",
|
||||
"snap/": "Kubernetes, Helm and Deployment",
|
||||
"terraform/": "Kubernetes, Helm and Deployment",
|
||||
"weed/": "Core and CLI",
|
||||
"test/": "Tests and CI",
|
||||
".github/": "Tests and CI",
|
||||
}
|
||||
|
||||
PR_LINE = re.compile(r"^\* (?P<title>.+?) by @(?P<author>\S+) in https://github\.com/\S+/pull/(?P<num>\d+)\s*$")
|
||||
# Lines regroup() writes between the PRs, so a second run accepts them.
|
||||
SCAFFOLD = re.compile(r"^$|^### (%s)$|^<details><summary>Dependencies \(\d+ updates?\)</summary>$|^</details>$"
|
||||
% "|".join(re.escape(s) for s in SECTIONS))
|
||||
# "fix(s3tables,s3vectors): ...", "s3 lifecycle: ...", "[Volume] ...", "Fix/admin ..."
|
||||
CONVENTIONAL = re.compile(r"^(?P<type>[a-z]+)\((?P<scope>[^)]*)\)!?$")
|
||||
TYPE_ONLY = {"fix", "feat", "feature", "perf", "refactor", "chore", "style", "revert", "improvement"}
|
||||
|
||||
|
||||
def scope_of(title):
|
||||
"""Return the raw scope text of a PR title, or ""."""
|
||||
t = title.strip()
|
||||
m = re.match(r"^\[([^\]]{1,40})\]", t)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.match(r"^(?:fix|feat|feature)/(\S+)", t, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
head, sep, _ = t.partition(":")
|
||||
if not sep or len(head) > 50:
|
||||
return ""
|
||||
head = head.strip()
|
||||
m = CONVENTIONAL.match(head.lower())
|
||||
if m:
|
||||
kind, scope = m.group("type"), m.group("scope")
|
||||
if kind in ("test", "tests", "ci"):
|
||||
return "test"
|
||||
if kind == "docs":
|
||||
return "docs"
|
||||
if kind == "build" and scope == "deps":
|
||||
return "build(deps)"
|
||||
return scope
|
||||
if head.lower() in TYPE_ONLY:
|
||||
return ""
|
||||
return head
|
||||
|
||||
|
||||
def section_by_scope(scope):
|
||||
if scope.strip().lower() == "build(deps)":
|
||||
return "Dependencies"
|
||||
for token in re.split(r"\s*[,+&]\s*|\s+and\s+", scope.lower()):
|
||||
token = token.strip()
|
||||
token = re.sub(r"^weed[/ ]", "", token)
|
||||
if not token:
|
||||
continue
|
||||
for pattern, section in SCOPE_RULES:
|
||||
if re.match(pattern, token):
|
||||
return section
|
||||
return ""
|
||||
|
||||
|
||||
def section_by_paths(paths):
|
||||
votes = {}
|
||||
code = [p for p in paths if not p.endswith(("_test.go", "_test.rs")) and not p.startswith("test/")]
|
||||
for p in code or paths:
|
||||
if p.endswith(".md"):
|
||||
section = "Documentation"
|
||||
else:
|
||||
best = max((pre for pre in PATH_RULES if p.startswith(pre)), key=len, default="")
|
||||
section = PATH_RULES.get(best, "Other")
|
||||
votes[section] = votes.get(section, 0) + 1
|
||||
if not votes:
|
||||
return "Other"
|
||||
# Most files wins; ties go to the earlier section in display order.
|
||||
return max(votes, key=lambda s: (votes[s], -SECTIONS.index(s)))
|
||||
|
||||
|
||||
def classify(title, author, pr_files):
|
||||
if author.startswith("dependabot") or title.lower().startswith("build(deps)"):
|
||||
return "Dependencies"
|
||||
section = section_by_scope(scope_of(title))
|
||||
if section:
|
||||
return section
|
||||
return section_by_paths(pr_files())
|
||||
|
||||
|
||||
def split_notes(body):
|
||||
"""Split generated notes into (PR lines, the text before them, the text after,
|
||||
the non-blank lines of What's Changed)."""
|
||||
lines = body.replace("\r\n", "\n").split("\n")
|
||||
start = next((i for i, l in enumerate(lines) if l.strip() == "## What's Changed"), None)
|
||||
if start is None:
|
||||
return [], body, "", []
|
||||
end = next((i for i in range(start + 1, len(lines))
|
||||
if lines[i].startswith(("## ", "**Full Changelog**"))), len(lines))
|
||||
prs, stray = [], []
|
||||
block = [l.rstrip() for l in lines[start + 1:end] if l.strip()]
|
||||
for line in lines[start + 1:end]:
|
||||
if PR_LINE.match(line):
|
||||
prs.append(line)
|
||||
elif not SCAFFOLD.match(line.strip()):
|
||||
stray.append(line)
|
||||
if prs and stray:
|
||||
raise ValueError("What's Changed has lines that are not PRs: %r" % stray[:3])
|
||||
before = "\n".join(lines[:start]).rstrip()
|
||||
after = "\n".join(lines[end:]).strip()
|
||||
return prs, before, after, block
|
||||
|
||||
|
||||
def regroup(body, pr_files):
|
||||
prs, before, after, block = split_notes(body)
|
||||
if not prs:
|
||||
return body
|
||||
groups = {s: [] for s in SECTIONS}
|
||||
for line in prs:
|
||||
m = PR_LINE.match(line)
|
||||
num = int(m.group("num"))
|
||||
groups[classify(m.group("title"), m.group("author"), lambda: pr_files(num))].append(line)
|
||||
|
||||
out = [before, ""] if before else []
|
||||
out.append("## What's Changed")
|
||||
for section in SECTIONS:
|
||||
items = groups[section]
|
||||
if not items:
|
||||
continue
|
||||
out.append("")
|
||||
if section == "Dependencies":
|
||||
out.append("<details><summary>Dependencies (%d update%s)</summary>" % (len(items), "" if len(items) == 1 else "s"))
|
||||
out.append("")
|
||||
out.extend(items)
|
||||
out.append("")
|
||||
out.append("</details>")
|
||||
else:
|
||||
out.append("### " + section)
|
||||
out.extend(items)
|
||||
if after:
|
||||
out.extend(["", after])
|
||||
notes = "\n".join(out).strip() + "\n"
|
||||
# Headings are accepted only as this script's own grouping: a hand-written
|
||||
# "### Security" over PRs of other components would otherwise be dropped.
|
||||
if len(block) != len(prs) and split_notes(notes)[3] != block:
|
||||
raise ValueError("What's Changed has headings that do not match the component grouping")
|
||||
return notes
|
||||
|
||||
|
||||
def gh(*args, stdin=None):
|
||||
return subprocess.run(["gh", *args], input=stdin, capture_output=True, text=True, check=True).stdout
|
||||
|
||||
|
||||
def pr_files(repo, num):
|
||||
return gh("pr", "view", str(num), "-R", repo, "--json", "files", "--jq", ".files[].path").splitlines()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--repo", default="seaweedfs/seaweedfs")
|
||||
ap.add_argument("--tag", required=True)
|
||||
ap.add_argument("--write", action="store_true", help="update the release notes on GitHub")
|
||||
args = ap.parse_args()
|
||||
|
||||
body = json.loads(gh("release", "view", args.tag, "-R", args.repo, "--json", "body"))["body"]
|
||||
|
||||
failed = []
|
||||
|
||||
def files_of(num):
|
||||
try:
|
||||
return pr_files(args.repo, num)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("warning: files of #%d: %s" % (num, e.stderr.strip()), file=sys.stderr)
|
||||
failed.append(num)
|
||||
return []
|
||||
|
||||
try:
|
||||
notes = regroup(body, files_of)
|
||||
except ValueError as e:
|
||||
sys.exit("release %s: %s" % (args.tag, e))
|
||||
sys.stdout.write(notes)
|
||||
if notes.strip() == body.replace("\r\n", "\n").strip():
|
||||
print("release %s: notes already grouped, nothing to change" % args.tag, file=sys.stderr)
|
||||
return
|
||||
if failed and args.write:
|
||||
sys.exit("release %s: not writing, could not read the files of %s"
|
||||
% (args.tag, ", ".join("#%d" % n for n in failed)))
|
||||
if args.write:
|
||||
gh("release", "edit", args.tag, "-R", args.repo, "--notes-file", "-", stdin=notes)
|
||||
print("release %s: notes updated" % args.tag, file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def load():
|
||||
path = Path(__file__).with_name("release_notes_by_component.py")
|
||||
spec = importlib.util.spec_from_file_location("release_notes_by_component", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
rn = load()
|
||||
|
||||
|
||||
def no_files(num):
|
||||
raise AssertionError("files of #%d should not be needed" % num)
|
||||
|
||||
|
||||
class ClassifyTest(unittest.TestCase):
|
||||
def check(self, title, section, author="someone", files=None):
|
||||
got = rn.classify(title, author, (lambda: files) if files is not None else (lambda: no_files(0)))
|
||||
self.assertEqual(got, section, title)
|
||||
|
||||
def test_title_prefixes(self):
|
||||
cases = {
|
||||
"s3: keep serving for -preStopSeconds after SIGTERM": "S3 API",
|
||||
"s3api: keep a multipart upload's state on the object's owner filer": "S3 API",
|
||||
"fix(s3api): reject an invalid storage class before uploading the body": "S3 API",
|
||||
"s3 lifecycle: count filer entries a walk reads but skips as progress": "S3 API",
|
||||
"iam/oidc: accept a list audience": "S3 API",
|
||||
"fix(s3tables,s3vectors): bind DeleteNamespace to the authorized records": "S3 Tables and Iceberg",
|
||||
"iceberg maintenance: expire snapshots": "S3 Tables and Iceberg",
|
||||
"s3vectors: support TagResource": "Vectors",
|
||||
"s3 vectors: page ListVectors": "Vectors",
|
||||
"vector: compact segments": "Vectors",
|
||||
"filer: fail health check when absent from lock ring": "Filer",
|
||||
"filer.sync: skip a rewritten source version": "Filer",
|
||||
"filer.remote.sync: retry": "Filer",
|
||||
"webdav: honor locks": "Filer",
|
||||
"remote.uncache, remote.meta.sync, remote.cache: keep local writes": "Filer",
|
||||
"mount: cache dir": "Mount",
|
||||
"fix(mount): flush on close": "Mount",
|
||||
"volume: keep reads over the download limit local": "Volume Server",
|
||||
"[Volume] Preserve request semantics in replica proxy reads": "Volume Server",
|
||||
"operation: pad the marginal gzip test": "Volume Server",
|
||||
"fix(storage): update deletion metrics": "Volume Server",
|
||||
"rust volume: bulk-load needle maps": "Rust Volume Server",
|
||||
"seaweed-volume: pin the status": "Rust Volume Server",
|
||||
"volume(rust): fix": "Rust Volume Server",
|
||||
"ec: mount shards through the encode's .vif": "Erasure Coding",
|
||||
"ec.rebuild: pick the right disk": "Erasure Coding",
|
||||
"ec_balance: spread shards": "Erasure Coding",
|
||||
"master: unregister a volume server whose heartbeats stop": "Master and Topology",
|
||||
"topology: never report negative free volume slots": "Master and Topology",
|
||||
"wdclient: let master-pushed adds replace locations": "Master and Topology",
|
||||
"admin: manage filer webhook targets": "Admin UI and Workers",
|
||||
"Fix/admin cluster submenu": "Admin UI and Workers",
|
||||
"weed-worker: retry": "Admin UI and Workers",
|
||||
"shell: fix s3.circuitBreaker help examples": "Shell",
|
||||
"volume.fsck: skip deleted": "Shell",
|
||||
"fix(fsck): inspect references to deleted volumes": "Shell",
|
||||
"security: optional dialed-host check": "Security",
|
||||
"helm: drop the 4.53 vector metrics render guard": "Kubernetes, Helm and Deployment",
|
||||
"install.sh: add --enterprise": "Kubernetes, Helm and Deployment",
|
||||
"mq/kafka: commit offsets": "Message Queue",
|
||||
"server: add -grpc.localSocketDir": "Core and CLI",
|
||||
"pb: recover when a local gRPC socket file is deleted": "Core and CLI",
|
||||
"docs: explain the 3-filer recommendation": "Documentation",
|
||||
"test(fuse): two mounts staging the same files": "Tests and CI",
|
||||
"test/s3vectors: a hardened cluster": "Tests and CI",
|
||||
"s3vectors test: cover tags": "Tests and CI",
|
||||
"ci: run the Python 3.9 jobs on 3.10": "Tests and CI",
|
||||
"build(deps): bump actions/setup-go from 6 to 7": "Dependencies",
|
||||
}
|
||||
for title, section in cases.items():
|
||||
self.check(title, section)
|
||||
|
||||
def test_dependabot_author(self):
|
||||
self.check("Bump x from 1 to 2", "Dependencies", author="dependabot[bot]")
|
||||
|
||||
def test_no_prefix_uses_files(self):
|
||||
self.check("Fix s3 cancelled read relookup", "S3 API",
|
||||
files=["weed/s3api/s3api_object_handlers.go", "weed/s3api/s3api_object_handlers_test.go"])
|
||||
self.check("Retry transient failures in star history generation", "Tests and CI",
|
||||
files=[".github/scripts/star_history.py"])
|
||||
self.check("fix: handle empty chunk list", "Filer",
|
||||
files=["weed/filer/filechunks.go", "weed/storage/needle/needle.go", "weed/filer/reader_at.go"])
|
||||
self.check("Improve everything", "Other", files=[])
|
||||
|
||||
def test_test_files_count_only_when_alone(self):
|
||||
self.check("Make it faster", "Mount",
|
||||
files=["weed/mount/weedfs.go", "test/fuse/a_test.go", "test/fuse/b_test.go"])
|
||||
self.check("Make it right", "S3 API",
|
||||
files=["weed/s3api/auth.go", "test/s3/iam/test_a.py", "test/s3/iam/test_b.py"])
|
||||
self.check("Cover more", "Tests and CI", files=["test/s3/iam/test_a.py"])
|
||||
|
||||
def test_path_with_space(self):
|
||||
with mock.patch.object(rn, "gh", return_value="test/s3/iam/policy cases.json\n"):
|
||||
files = rn.pr_files("seaweedfs/seaweedfs", 1)
|
||||
self.assertEqual(files, ["test/s3/iam/policy cases.json"])
|
||||
self.check("Cover more", "Tests and CI", files=files)
|
||||
|
||||
|
||||
NOTES = """## What's Changed
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
"""
|
||||
|
||||
|
||||
class RegroupTest(unittest.TestCase):
|
||||
def files(self, num):
|
||||
self.assertEqual(num, 12600)
|
||||
return ["weed/s3api/s3api_object_handlers.go"]
|
||||
|
||||
def test_regroup(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(out, """## What's Changed
|
||||
|
||||
### S3 API
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
### Filer
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
|
||||
<details><summary>Dependencies (1 update)</summary>
|
||||
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
|
||||
</details>
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
""")
|
||||
|
||||
def test_idempotent(self):
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(rn.regroup(once, self.files), once)
|
||||
|
||||
def test_every_pr_kept(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
for n in (12709, 12707, 12685, 12366, 12600):
|
||||
self.assertEqual(out.count("/pull/%d\n" % n), 2 if n == 12600 else 1)
|
||||
|
||||
def test_changelog_kept_without_new_contributors(self):
|
||||
body = NOTES.replace(
|
||||
"## New Contributors\n* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600\n\n", "")
|
||||
out = rn.regroup(body, self.files)
|
||||
self.assertTrue(out.endswith("</details>\n\n**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54\n"), out)
|
||||
self.assertEqual(rn.regroup(out, self.files), out)
|
||||
|
||||
def test_unknown_line_refused(self):
|
||||
body = NOTES.replace("## New Contributors", "* a hand-written note\n\n## New Contributors")
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(body, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Known issue\n"), self.files)
|
||||
|
||||
def test_hand_written_component_heading_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Security\n"), self.files)
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(once.replace("\n### Filer\n", "\n ### Filer\n"), self.files)
|
||||
|
||||
def test_author_from_the_by_clause(self):
|
||||
body = "## What's Changed\n* Bump a by @dependabot[bot] in docs by @alice in https://github.com/seaweedfs/seaweedfs/pull/7\n"
|
||||
self.assertIn("### Documentation", rn.regroup(body, lambda n: ["README.md"]))
|
||||
|
||||
def test_notes_without_prs_unchanged(self):
|
||||
body = "Hand-written notes\n\n**Full Changelog**: x"
|
||||
self.assertEqual(rn.regroup(body, no_files), body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -5,11 +5,13 @@ on:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
@@ -28,3 +30,12 @@ jobs:
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
|
||||
release-notes-by-component:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run release notes grouping unit tests
|
||||
run: python3 .github/scripts/test_release_notes_by_component.py
|
||||
@@ -259,40 +259,6 @@ jobs:
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "Security configuration renders correctly"
|
||||
|
||||
echo "=== Testing volume server HTTPS ==="
|
||||
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/security.yaml | grep -q 'enabled = false'
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
--set global.seaweedfs.volumeHttps.enabled=true \
|
||||
--set volumes.hot.replicas=1 \
|
||||
> /tmp/volume-https.yaml
|
||||
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml > /tmp/volume-https-client.txt
|
||||
grep -q 'enabled = true' /tmp/volume-https-client.txt
|
||||
grep -q 'cert = "/usr/local/share/ca-certificates/client/tls.crt"' /tmp/volume-https-client.txt
|
||||
grep -q 'ca = "/usr/local/share/ca-certificates/ca/tls.crt"' /tmp/volume-https-client.txt
|
||||
awk '/^ *\[https.volume\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml \
|
||||
| grep -q 'cert = "/usr/local/share/ca-certificates/volume/tls.crt"'
|
||||
grep -q "'\*.test-seaweedfs-volume.default'" /tmp/volume-https.yaml
|
||||
grep -q "'\*.test-seaweedfs-volume-hot.default'" /tmp/volume-https.yaml
|
||||
test "$(grep -c 'scheme: HTTPS' /tmp/volume-https.yaml)" -eq 4
|
||||
test "$(grep -c 'checksum/volume-https:' /tmp/volume-https.yaml)" -eq 4
|
||||
if grep -q 'checksum/volume-https' /tmp/security.yaml; then
|
||||
echo "FAIL: the volume HTTPS rollout annotation should only render while the option is on"
|
||||
exit 1
|
||||
fi
|
||||
if helm template test $CHART_DIR --set global.seaweedfs.volumeHttps.enabled=true > /dev/null 2>/tmp/volume-https-nosec.err; then
|
||||
echo "FAIL: volumeHttps without enableSecurity should fail to render"
|
||||
exit 1
|
||||
fi
|
||||
grep -q "requires global.seaweedfs.enableSecurity" /tmp/volume-https-nosec.err
|
||||
if helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true \
|
||||
--set global.seaweedfs.volumeHttps.enabled=true --set allInOne.enabled=true > /dev/null 2>/tmp/volume-https-aio.err; then
|
||||
echo "FAIL: volumeHttps with allInOne should fail to render"
|
||||
exit 1
|
||||
fi
|
||||
grep -q "not supported with allInOne" /tmp/volume-https-aio.err
|
||||
echo "Volume server HTTPS renders [https.volume], [https.client], certificate names, HTTPS probes and rollout annotations"
|
||||
|
||||
echo "=== Testing secure bucket-creation hook certificate mounts ==="
|
||||
helm template test $CHART_DIR \
|
||||
--show-only templates/shared/post-install-bucket-hook.yaml \
|
||||
@@ -394,9 +360,9 @@ jobs:
|
||||
def docs(manifest):
|
||||
return [d for d in yaml.safe_load_all(manifest) if d]
|
||||
|
||||
def security_config(manifest, name):
|
||||
def configmap(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") == "Secret" and d["metadata"]["name"] == name:
|
||||
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
|
||||
return d
|
||||
return None
|
||||
|
||||
@@ -421,13 +387,13 @@ jobs:
|
||||
# security-related; preserve that so this PR is non-breaking on
|
||||
# existing installs.
|
||||
out = render({})
|
||||
if security_config(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security Secret should not render")
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security ConfigMap should not render")
|
||||
else:
|
||||
print("defaults: no security-config Secret (unchanged)")
|
||||
print("defaults: no security-config ConfigMap (unchanged)")
|
||||
|
||||
# Case 2: filerWrite=true alone is the documented opt-in for
|
||||
# the Admin UI Users tab. The Secret must render with
|
||||
# the Admin UI Users tab. Configmap must render with
|
||||
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
|
||||
# only exist with mTLS).
|
||||
out = render({
|
||||
@@ -437,11 +403,11 @@ jobs:
|
||||
# refuses to bind without authentication.
|
||||
"admin.secret.adminPassword": "ci-admin-password",
|
||||
})
|
||||
cm = security_config(out, "test-seaweedfs-security-config")
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("filerWrite=true: security Secret missing")
|
||||
failed.append("filerWrite=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["stringData"]["security.toml"]
|
||||
toml = cm["data"]["security.toml"]
|
||||
if "[jwt.filer_signing]" not in toml:
|
||||
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
|
||||
if "[grpc" in toml:
|
||||
@@ -449,7 +415,7 @@ jobs:
|
||||
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
|
||||
print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
|
||||
# Case 3: filer + admin pods must MOUNT the security Secret
|
||||
# Case 3: filer + admin pods must MOUNT the security ConfigMap
|
||||
# under filerWrite=true so the JWT key reaches both processes.
|
||||
# Cert volumes must NOT be present (no mTLS).
|
||||
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
|
||||
@@ -469,11 +435,11 @@ jobs:
|
||||
# with both [jwt.signing] and [grpc.*]. Guards against the
|
||||
# decoupling change accidentally regressing the mTLS path.
|
||||
out = render({"global.seaweedfs.enableSecurity": "true"})
|
||||
cm = security_config(out, "test-seaweedfs-security-config")
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("enableSecurity=true: security Secret missing")
|
||||
failed.append("enableSecurity=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["stringData"]["security.toml"]
|
||||
toml = cm["data"]["security.toml"]
|
||||
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
|
||||
if missing:
|
||||
failed.append(f"enableSecurity=true: security.toml missing {missing}")
|
||||
@@ -490,8 +456,8 @@ jobs:
|
||||
except subprocess.CalledProcessError as e:
|
||||
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
|
||||
continue
|
||||
if security_config(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render the security Secret")
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render configmap")
|
||||
else:
|
||||
print(f"{null_path}=null: render tolerates explicit null")
|
||||
|
||||
@@ -595,16 +561,16 @@ jobs:
|
||||
mount = next((m for m in pod["containers"][0].get("volumeMounts", []) if m["name"] == "security-config"), None)
|
||||
# The file has to land on one of weed's config search paths, so
|
||||
# the target matters as much as the volume itself - and a volume
|
||||
# of that name backed by anything but the chart's Secret
|
||||
# of that name backed by anything but the chart's ConfigMap
|
||||
# leaves `weed shell` without the signing key just the same.
|
||||
target = ("/etc/seaweedfs/security.toml", "security.toml")
|
||||
secret = "test-seaweedfs-security-config"
|
||||
configmap = "test-seaweedfs-security-config"
|
||||
if vol is None or mount is None:
|
||||
failed.append("filerWrite=true: bucket hook does not mount security-config (s3.configure would fail)")
|
||||
elif (mount.get("mountPath"), mount.get("subPath")) != target:
|
||||
failed.append(f"filerWrite=true: bucket hook mounts security-config as {mount}")
|
||||
elif vol.get("secret", {}).get("secretName") != secret:
|
||||
failed.append(f"filerWrite=true: security-config is not backed by the {secret} Secret: {vol}")
|
||||
elif vol.get("configMap", {}).get("name") != configmap:
|
||||
failed.append(f"filerWrite=true: security-config is not backed by the {configmap} ConfigMap: {vol}")
|
||||
else:
|
||||
print("filerWrite=true: bucket hook mounts security-config at /etc/seaweedfs/security.toml")
|
||||
|
||||
@@ -2110,7 +2076,7 @@ jobs:
|
||||
"--set", "admin.enabled=true", "--set", "admin.secret.adminPassword=ci",
|
||||
"--set", "cosi.enabled=true", "--set", "networkPolicy.enabled=true",
|
||||
"--set", "volumes.dc2.replicas=1",
|
||||
# render the security Secret without its generated jwt key
|
||||
# render the security ConfigMap without its generated jwt key
|
||||
"--set", "global.seaweedfs.enableSecurity=true",
|
||||
"--set", "global.seaweedfs.securityConfig.jwtSigning.volumeWrite=false",
|
||||
]
|
||||
|
||||
@@ -4,7 +4,9 @@ name: "release: bump version and cut the release"
|
||||
# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master
|
||||
# 2. push the <appVersion> tag, which fans out to the workflows that trigger on
|
||||
# `push: tags` (binaries_release*, container_release_unified, helm_manual_release)
|
||||
# 3. create the GitHub release, with GitHub's generated notes
|
||||
# 3. create the GitHub release, with GitHub's generated notes, then regroup those
|
||||
# notes by component (S3 API, Filer, Mount, ...) with
|
||||
# .github/scripts/release_notes_by_component.py
|
||||
# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator,
|
||||
# which pick up the new master through `go get -u`, and wait for both
|
||||
#
|
||||
@@ -41,6 +43,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read # the notes step reads a PR's files when its title has no component prefix
|
||||
outputs:
|
||||
app_version: ${{ steps.compute.outputs.app_version }}
|
||||
sha: ${{ steps.tag.outputs.sha }}
|
||||
@@ -174,6 +177,19 @@ jobs:
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag
|
||||
|
||||
# GitHub's generated notes are one flat list of PRs. Sort them under
|
||||
# component headings by title prefix (or changed files). The release is
|
||||
# already published, so a failure here only leaves the flat list; rerun
|
||||
# the script by hand to fix it:
|
||||
# python3 .github/scripts/release_notes_by_component.py --tag <tag> --write
|
||||
- name: Group the release notes by component
|
||||
if: ${{ !inputs.dry_run }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: python3 .github/scripts/release_notes_by_component.py --repo "$GITHUB_REPOSITORY" --tag "$TAG" --write > /dev/null
|
||||
|
||||
downstream:
|
||||
needs: release
|
||||
if: ${{ inputs.downstream && !inputs.dry_run }}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
name: "S3 listing study tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/listing_study/**'
|
||||
- '.github/workflows/s3-listing-study-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number || github.ref }}/s3-listing-study-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
listing-study:
|
||||
name: s3-listing-study tools and strategies
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
defaults:
|
||||
run:
|
||||
working-directory: test/s3/listing_study
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Cache pinned listing tools
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: test/s3/listing_study/.work/tools
|
||||
key: listing-study-tools-${{ runner.os }}-${{ hashFiles('test/s3/listing_study/Makefile', 'test/s3/listing_study/install_tools.py') }}
|
||||
|
||||
- name: Build weed and install the study's pinned tools
|
||||
run: make build-weed tools
|
||||
|
||||
- name: Run the listing study against SeaweedFS
|
||||
timeout-minutes: 15
|
||||
run: make test
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
tail -200 .work/weed.log || true
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-listing-study-results
|
||||
path: |
|
||||
test/s3/listing_study/.work/run/results.json
|
||||
test/s3/listing_study/.work/weed.log
|
||||
retention-days: 14
|
||||
@@ -68,15 +68,7 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache suite Maven dependencies
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: snowflake-s3compat-m2-${{ hashFiles('test/s3/snowflake/run.sh') }}
|
||||
restore-keys: |
|
||||
snowflake-s3compat-m2-
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
@@ -86,8 +78,6 @@ jobs:
|
||||
|
||||
- name: Run Snowflake S3Compat API tests
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
MAVEN_ARGS: -ntp -s ${{ github.workspace }}/test/s3/snowflake/maven-settings.xml
|
||||
run: |
|
||||
# Starts weed server, creates the buckets/objects the suite needs,
|
||||
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
|
||||
|
||||
@@ -684,12 +684,6 @@ jobs:
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
|
||||
- name: Cache Iceberg jars
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/seaweedfs-test-jars
|
||||
key: iceberg-jars-${{ hashFiles('test/s3tables/catalog_spark/setup_test.go') }}
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
name: "S3 Tigris Compatibility Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/tigris_compat/**'
|
||||
- '.github/workflows/s3-tigris-compat-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/tigris_compat/**'
|
||||
- '.github/workflows/s3-tigris-compat-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-tigris-compat-tests:
|
||||
name: s3-api-compat-tests (Tigris)
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run s3-api-compat-tests
|
||||
working-directory: test/s3/tigris_compat
|
||||
run: make test
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/tigris_compat
|
||||
run: tail -200 weed-test.log || true
|
||||
|
||||
- name: Upload report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tigris-compat-results
|
||||
path: test/s3/tigris_compat/results.json
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
@@ -6,16 +6,12 @@ on:
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -56,12 +56,8 @@ jobs:
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
CADDY_VERSION=2.11.9
|
||||
base="https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}"
|
||||
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_checksums.txt"
|
||||
grep " caddy_${CADDY_VERSION}_linux_amd64.tar.gz$" "caddy_${CADDY_VERSION}_checksums.txt" | sha512sum -c -
|
||||
tar -xzf "caddy_${CADDY_VERSION}_linux_amd64.tar.gz" caddy
|
||||
curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
|
||||
chmod +x caddy
|
||||
./caddy version
|
||||
echo "{
|
||||
auto_https disable_redirects
|
||||
|
||||
@@ -30,12 +30,10 @@ so your deployment will be spread/HA.
|
||||
|
||||
The chart's templates render with Helm 3.16.3 and newer versions tested in CI.
|
||||
Earlier Helm versions are not covered by this chart's compatibility checks.
|
||||
When JWT signing is enabled, `security.toml` is kept in the
|
||||
`<fullname>-security-config` Secret. A live upgrade reuses keys from that
|
||||
Secret, or from the security ConfigMap of earlier chart versions (including
|
||||
the legacy ConfigMap name); a key absent from them is generated. Plain
|
||||
`helm template` does not read cluster state; use an install and upgrade
|
||||
against a cluster to check key persistence.
|
||||
When JWT signing is enabled, a live upgrade reuses keys from the existing
|
||||
security ConfigMap (including the legacy ConfigMap name); a key absent from
|
||||
that ConfigMap is generated. Plain `helm template` does not read cluster state;
|
||||
use an install and upgrade against a cluster to check key persistence.
|
||||
|
||||
The key reader supports single-line quoted TOML strings in JWT sections,
|
||||
including indented assignments, quoted key names, and bare or simply quoted
|
||||
@@ -571,8 +569,6 @@ s3:
|
||||
|
||||
With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
|
||||
|
||||
Outside all-in-one mode, the volume Ingress and the default volume route target the `client-volume` Service, which lists only Ready volume pods. The `volume` Service also publishes pods that are not Ready yet, so their DNS records can exist before readiness. An explicit `backendRefs` entry that names the `volume` Service can therefore reach a volume server before it is Ready. Name the `client-volume` Service instead, with `volume.port`, and copy its `metadata.name` from `helm template` because long release names are truncated.
|
||||
|
||||
## Network Policies
|
||||
|
||||
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
|
||||
@@ -614,17 +610,6 @@ Two things worth knowing before you turn this on:
|
||||
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
|
||||
## Volume server HTTPS
|
||||
|
||||
With `global.seaweedfs.enableSecurity`, gRPC between components uses mutual TLS, but the volume servers' HTTP port, which carries the file data, stays plaintext. Set `global.seaweedfs.volumeHttps.enabled: true` to serve it over TLS:
|
||||
|
||||
- `[https.volume]` points at the volume certificate, which then also names every volume group's service (`*.<fullname>-volume.<namespace>`), since volume servers register as `<pod>.<fullname>-volume.<namespace>`. With `certificates.externalCertificates`, the provided `<fullname>-volume-cert` needs those names.
|
||||
- `[https.client]` is enabled with the client certificate and the chart CA, so every chart pod that talks to the volume servers does it over HTTPS.
|
||||
- The volume probes use HTTPS.
|
||||
- Every pod that reads `security.toml` gets a `checksum/volume-https` annotation, so switching the option on or off restarts them together.
|
||||
|
||||
The volume servers do not require client certificates, because kubelet probes cannot present one; enable JWT signing (`securityConfig.jwtSigning`) to authorize volume reads and writes. `[https.client]` also sends the admin UI's HTTP lookups on masters and S3 over HTTPS, which their plain listeners refuse, so the UI can miss volume public URLs and refuse SSE-S3 vector buckets. A volume Ingress or HTTPRoute needs its controller's HTTPS-backend setting. `weed mount`, the CSI driver and other clients outside the chart need their own `[https.client]` with the chart CA. The option is not supported with `allInOne`, whose volume server registers by pod IP.
|
||||
|
||||
## Common labels
|
||||
|
||||
`commonLabels` adds labels to the metadata of every object the chart creates. It may be a map or a template string rendering to one. The chart's own labels cannot be overridden, and pods (see `podLabels`) and `volumeClaimTemplates` are left unchanged.
|
||||
|
||||
@@ -56,7 +56,6 @@ spec:
|
||||
{{- with .Values.admin.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.admin.restartPolicy }}
|
||||
{{- if .Values.admin.affinity }}
|
||||
@@ -314,8 +313,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
{{- fail "global.seaweedfs.volumeHttps is not supported with allInOne: its volume server registers by pod IP, which the volume certificate cannot name" }}
|
||||
{{- end }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
|
||||
@@ -507,8 +504,8 @@ spec:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -38,14 +38,6 @@ spec:
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc.cluster.local'
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
{{- range $vname, $_ := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- $volumeService := printf "%s.%s" (include "seaweedfs.componentName" (list $ (trimSuffix "-" (printf "volume-%s" $vname)))) (include "seaweedfs.namespace" $) }}
|
||||
- '*.{{ $volumeService }}'
|
||||
- '*.{{ $volumeService }}.svc'
|
||||
- '*.{{ $volumeService }}.svc.cluster.local'
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
|
||||
@@ -39,7 +39,6 @@ spec:
|
||||
{{- with .Values.cosi.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.cosi.restartPolicy }}
|
||||
{{- if .Values.cosi.affinity }}
|
||||
@@ -196,8 +195,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -57,7 +57,6 @@ spec:
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.filer.restartPolicy }}
|
||||
{{- if .Values.filer.affinity }}
|
||||
@@ -453,8 +452,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -48,7 +48,6 @@ spec:
|
||||
{{- $_ := set $podAnnotations "checksum/master-config" (include "seaweedfs.master.configChecksum" .) }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.master.restartPolicy }}
|
||||
{{- if .Values.master.affinity }}
|
||||
@@ -300,8 +299,8 @@ spec:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -54,7 +54,6 @@ spec:
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.s3.restartPolicy }}
|
||||
{{- if .Values.s3.affinity }}
|
||||
@@ -289,8 +288,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -43,7 +43,6 @@ spec:
|
||||
{{- with .Values.sftp.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.sftp.restartPolicy }}
|
||||
{{- if .Values.sftp.affinity }}
|
||||
@@ -298,8 +297,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -491,25 +491,6 @@ true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when the volume servers serve HTTPS. Their certificates come from
|
||||
enableSecurity, so the option fails without it. */}}
|
||||
{{- define "seaweedfs.volumeHttps" -}}
|
||||
{{- if ((.Values.global.seaweedfs).volumeHttps).enabled -}}
|
||||
{{- if not .Values.global.seaweedfs.enableSecurity -}}
|
||||
{{- fail "global.seaweedfs.volumeHttps.enabled requires global.seaweedfs.enableSecurity, which provides the certificates" -}}
|
||||
{{- end -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Pod annotation that restarts the readers of security.toml when volume
|
||||
HTTPS is switched on or off. */}}
|
||||
{{- define "seaweedfs.volumeHttpsAnnotation" -}}
|
||||
{{- with include "seaweedfs.volumeHttps" . -}}
|
||||
checksum/volume-https: {{ sha256sum . | quote }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Read a JWT key from the chart's existing security.toml without fromToml
|
||||
(which requires Helm >=3.17). Args: (list "<section>" $raw).
|
||||
Return the single-line TOML string token, including its quotes, so escapes
|
||||
|
||||
@@ -286,8 +286,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
@@ -11,8 +11,7 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
data:
|
||||
{{- $fullname := include "seaweedfs.fullname" . }}
|
||||
{{- $securityConfigName := printf "%s-security-config" $fullname }}
|
||||
{{- $existing := lookup "v1" "ConfigMap" (include "seaweedfs.namespace" .) $securityConfigName }}
|
||||
@@ -20,11 +19,7 @@ stringData:
|
||||
{{- $legacyName := printf "%s-%s" (include "seaweedfs.name" .) "security-config" }}
|
||||
{{- $existing = lookup "v1" "ConfigMap" (include "seaweedfs.namespace" .) $legacyName }}
|
||||
{{- end }}
|
||||
{{- $ownConfigMap := eq (dig "metadata" "annotations" "meta.helm.sh/release-name" "" $existing) .Release.Name }}
|
||||
{{- if and $ownConfigMap (or (and .Values.master.enabled (gt (int .Values.master.updatePartition) 0)) (and .Values.filer.enabled (gt (int .Values.filer.updatePartition) 0))) }}
|
||||
{{- fail "this upgrade moves security.toml from a ConfigMap to a Secret and removes the ConfigMap; run it with master.updatePartition=0 and filer.updatePartition=0 so no pod is left on the old template" }}
|
||||
{{- end }}
|
||||
{{- $existingToml := dig "data" "security.toml" "" (lookup "v1" "Secret" (include "seaweedfs.namespace" .) $securityConfigName) | b64dec | default (dig "data" "security.toml" "" $existing) }}
|
||||
{{- $existingToml := dig "data" "security.toml" "" $existing }}
|
||||
{{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
|
||||
{{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
|
||||
{{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
|
||||
@@ -35,7 +30,7 @@ stringData:
|
||||
# the jwt signing key is read by master and volume server
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
[jwt.signing]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 64 | quote) }}
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
{{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
|
||||
{{- end }}
|
||||
@@ -47,7 +42,7 @@ stringData:
|
||||
# - the Volume server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
[jwt.signing.read]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 64 | quote) }}
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
{{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
|
||||
{{- end }}
|
||||
@@ -59,7 +54,7 @@ stringData:
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
[jwt.filer_signing]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 64 | quote) }}
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
{{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
|
||||
{{- end }}
|
||||
@@ -71,7 +66,7 @@ stringData:
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
[jwt.filer_signing.read]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 64 | quote) }}
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
{{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
|
||||
{{- end }}
|
||||
@@ -126,25 +121,13 @@ stringData:
|
||||
cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
|
||||
[https.client]
|
||||
enabled = true
|
||||
cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
ca = "/usr/local/share/ca-certificates/ca/tls.crt"
|
||||
|
||||
[https.volume]
|
||||
cert = "/usr/local/share/ca-certificates/volume/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/volume/tls.key"
|
||||
{{- else }}
|
||||
|
||||
# volume server https: set global.seaweedfs.volumeHttps.enabled
|
||||
# volume server https options
|
||||
# Note: work in progress!
|
||||
# this does not work with other clients, e.g., "weed filer|mount" etc, yet.
|
||||
[https.client]
|
||||
enabled = false
|
||||
[https.volume]
|
||||
cert = ""
|
||||
key = ""
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -25,7 +25,6 @@ spec:
|
||||
annotations:
|
||||
{{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }}
|
||||
@@ -151,8 +150,8 @@ spec:
|
||||
emptyDir: {}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- range list "ca" "client" "vector" }}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $volumeEnabled .Values.volume.ingress.enabled }}
|
||||
{{- /* Determine service name based on deployment mode */}}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
{{- /* Ready-only backend for the volume ingress and HTTPRoute; the volume Service publishes not-ready pods for DNS.
|
||||
Not named volume-client: every volume-<key> name belongs to a group under .Values.volumes. */}}
|
||||
{{- if and .Values.volume.enabled (not .Values.allInOne.enabled) (or .Values.volume.ingress.enabled .Values.volume.httpRoute.enabled) }}
|
||||
{{- $name := include "seaweedfs.componentName" (list . "client-volume") }}
|
||||
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- range $vname, $volume := $volumes }}
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
|
||||
{{- if and $volume.enabled (eq (include "seaweedfs.componentName" (list $ $volumeName)) $name) }}
|
||||
{{- fail (printf "The volume Ingress/HTTPRoute Service %q has the same name as the %s Service after truncation. Shorten fullnameOverride or the release name." $name $volumeName) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
|
||||
{{- if .Values.volume.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.volume.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- /* No metrics port, so the volume ServiceMonitor does not scrape each pod twice */}}
|
||||
ports:
|
||||
- name: "swfs-volume"
|
||||
port: {{ .Values.volume.port }}
|
||||
targetPort: {{ .Values.volume.port }}
|
||||
protocol: TCP
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
{{- end }}
|
||||
@@ -23,9 +23,6 @@ metadata:
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
{{- /* The master hands out ${POD_NAME}.<this service> as soon as a volume server heartbeats, before it is Ready.
|
||||
Ingress and HTTPRoute use the client-volume Service instead, which publishes only Ready pods. */}}
|
||||
publishNotReadyAddresses: true
|
||||
internalTrafficPolicy: {{ $volume.internalTrafficPolicy | default "Cluster" }}
|
||||
ports:
|
||||
- name: "swfs-volume"
|
||||
|
||||
@@ -55,7 +55,6 @@ spec:
|
||||
{{- with $volume.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
{{- if $volume.affinity }}
|
||||
affinity:
|
||||
@@ -282,7 +281,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ $volume.readinessProbe.httpGet.path }}
|
||||
port: {{ $volume.port }}
|
||||
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.readinessProbe.httpGet.scheme }}{{ end }}
|
||||
scheme: {{ $volume.readinessProbe.httpGet.scheme }}
|
||||
initialDelaySeconds: {{ $volume.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ $volume.readinessProbe.periodSeconds }}
|
||||
successThreshold: {{ $volume.readinessProbe.successThreshold }}
|
||||
@@ -294,7 +293,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ $volume.livenessProbe.httpGet.path }}
|
||||
port: {{ $volume.port }}
|
||||
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.livenessProbe.httpGet.scheme }}{{ end }}
|
||||
scheme: {{ $volume.livenessProbe.httpGet.scheme }}
|
||||
initialDelaySeconds: {{ $volume.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ $volume.livenessProbe.periodSeconds }}
|
||||
successThreshold: {{ $volume.livenessProbe.successThreshold }}
|
||||
@@ -371,8 +370,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" $ }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" $ }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" $ }}-security-config
|
||||
{{- end }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -46,7 +46,6 @@ spec:
|
||||
{{- with .Values.worker.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.worker.restartPolicy }}
|
||||
{{- if .Values.worker.affinity }}
|
||||
@@ -375,8 +374,8 @@ spec:
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findDocument(documents []map[string]interface{}, kind, name string) map[string]interface{} {
|
||||
for _, document := range documents {
|
||||
if document["kind"] == kind && document["metadata"].(map[string]interface{})["name"] == name {
|
||||
return document
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestVolumeServicePublishesNotReadyPods(t *testing.T) {
|
||||
_, documents := render(t)
|
||||
service := findDocument(documents, "Service", "vector-test-seaweedfs-volume")
|
||||
if service == nil {
|
||||
t.Fatal("no volume service")
|
||||
}
|
||||
if service["spec"].(map[string]interface{})["publishNotReadyAddresses"] != true {
|
||||
t.Fatal("volume pod DNS names must resolve before the pods are Ready")
|
||||
}
|
||||
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
|
||||
t.Fatal("client-volume service rendered without ingress or httpRoute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeIngressUsesReadyOnlyService(t *testing.T) {
|
||||
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volume.metricsPort=9327")
|
||||
client := findDocument(documents, "Service", "vector-test-seaweedfs-client-volume")
|
||||
if client == nil {
|
||||
t.Fatal("no client-volume service")
|
||||
}
|
||||
spec := client["spec"].(map[string]interface{})
|
||||
if _, found := spec["publishNotReadyAddresses"]; found {
|
||||
t.Fatal("client-volume must publish only Ready pods")
|
||||
}
|
||||
for _, port := range spec["ports"].([]interface{}) {
|
||||
if port.(map[string]interface{})["name"] == "metrics" {
|
||||
t.Fatal("client-volume must not expose metrics, or the ServiceMonitor scrapes each pod twice")
|
||||
}
|
||||
}
|
||||
|
||||
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
|
||||
if ingress == nil {
|
||||
t.Fatal("no volume ingress")
|
||||
}
|
||||
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
|
||||
t.Fatalf("volume ingress backend is %v", backend)
|
||||
}
|
||||
|
||||
route := findDocument(documents, "HTTPRoute", "vector-test-seaweedfs-volume")
|
||||
if route == nil {
|
||||
t.Fatal("no volume httpRoute")
|
||||
}
|
||||
rule := route["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := rule["backendRefs"].([]interface{})[0].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" {
|
||||
t.Fatalf("volume httpRoute backend is %v", backend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeRoutingServiceDoesNotCollideWithVolumeGroups(t *testing.T) {
|
||||
_, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volumes.client.replicas=1")
|
||||
seen := map[string]bool{}
|
||||
for _, document := range documents {
|
||||
key := fmt.Sprint(document["kind"], "/", document["metadata"].(map[string]interface{})["name"])
|
||||
if seen[key] {
|
||||
t.Fatalf("%s rendered twice", key)
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
if !seen["Service/vector-test-seaweedfs-volume-client"] || !seen["Service/vector-test-seaweedfs-client-volume"] {
|
||||
t.Fatal("volumes.client group Service and the routing Service must both render")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeRoutingServiceRejectsTruncatedCollision(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
fullname := strings.Repeat("a", 49) + "-client"
|
||||
output, err := exec.Command("helm", "template", "test", "..", "--set", "fullnameOverride="+fullname, "--set", "volume.ingress.enabled=true").CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(output), "same name as the volume Service") {
|
||||
t.Fatalf("colliding routing Service accepted: %v %s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeIngressAllInOne(t *testing.T) {
|
||||
_, documents := render(t, "allInOne.enabled=true", "volume.ingress.enabled=true")
|
||||
if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil {
|
||||
t.Fatal("client-volume service rendered in all-in-one mode")
|
||||
}
|
||||
ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume")
|
||||
if ingress == nil {
|
||||
t.Fatal("no volume ingress")
|
||||
}
|
||||
path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{})
|
||||
if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-all-in-one" {
|
||||
t.Fatalf("all-in-one volume ingress backend is %v", backend)
|
||||
}
|
||||
}
|
||||
@@ -45,18 +45,6 @@ global:
|
||||
volumeRead: 0
|
||||
filerWrite: 0
|
||||
filerRead: 0
|
||||
# Serve the volume servers' HTTP port over TLS with the chart's volume
|
||||
# certificate, and make every chart component reach them over HTTPS
|
||||
# ([https.volume] and [https.client] in security.toml). Requires
|
||||
# enableSecurity and is not supported with allInOne. Clients outside the
|
||||
# chart (weed mount, the CSI driver) need their own [https.client].
|
||||
# The volume servers do not ask for client certificates, because kubelet
|
||||
# probes cannot present one; JWT signing keeps authorizing volume access.
|
||||
# [https.client] also sends the admin UI's HTTP lookups on masters and
|
||||
# S3 over HTTPS, which their plain listeners refuse, so the UI can miss
|
||||
# volume public URLs and refuse SSE-S3 vector buckets.
|
||||
volumeHttps:
|
||||
enabled: false
|
||||
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
|
||||
serviceAccountName: "seaweedfs"
|
||||
serviceAccountAnnotations: {}
|
||||
|
||||
+1067
-1040
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 54 KiB |
@@ -156,14 +156,11 @@ public class SeaweedWrite {
|
||||
|
||||
entry.clearChunks();
|
||||
entry.addAllChunks(chunks);
|
||||
FilerProto.CreateEntryResponse response = filerClient.getBlockingStub().createEntry(
|
||||
filerClient.getBlockingStub().createEntry(
|
||||
FilerProto.CreateEntryRequest.newBuilder()
|
||||
.setDirectory(parentDirectory)
|
||||
.setEntry(entry)
|
||||
.build());
|
||||
if (!Strings.isNullOrEmpty(response.getError())) {
|
||||
throw new IOException(response.getError());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -262,8 +262,6 @@ message CreateEntryRequest {
|
||||
// Set on a create a filer forwarded to the entry's ring owner, so the owner
|
||||
// applies it locally instead of forwarding again. Clients leave it unset.
|
||||
bool is_moved = 8;
|
||||
// The store write must be durable before the reply, as for ?fsync=true.
|
||||
bool fsync = 9;
|
||||
}
|
||||
|
||||
// WriteCondition is the precondition the filer evaluates against the existing
|
||||
|
||||
@@ -2302,113 +2302,6 @@
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Writable Volumes per Disk",
|
||||
"description": "Disks holding more than one writable volume of a collection, and the most writable volumes of a collection on one disk.",
|
||||
"type": "timeseries",
|
||||
"id": 262,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "count by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}) > 1)",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{collection}} disks with 2+"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (collection) (sum by (cluster, collection, data_center, rack, node, disk_id) (SeaweedFS_master_writable_volumes_per_disk{cluster=~\"$cluster\", state=\"writable\"}))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "{{collection}} max on one disk"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -3442,102 +3335,6 @@
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Vacuum Dropped Needles",
|
||||
"description": "Live needles vacuum dropped because their data could not be read back intact (CRC mismatch, malformed or short record), counted since the volume server started. Any non-zero value means a lost replica copy; the volume server log has each dropped key.",
|
||||
"type": "timeseries",
|
||||
"id": 267,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 54
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (instance, collection) (SeaweedFS_volumeServer_vacuuming_dropped_needles{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{instance}} {{collection}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -6832,423 +6629,6 @@
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribe Delivery Lag",
|
||||
"description": "Seconds from a metadata event's mutation until the filer sends it to a subscriber, by stream scope",
|
||||
"type": "timeseries",
|
||||
"id": 263,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_delivery_lag_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Watermark Hold Duration",
|
||||
"description": "How long an aggregated read stayed held at one entry until the peers' watermark released it (memory: delivery watermark, disk: flush watermark)",
|
||||
"type": "timeseries",
|
||||
"id": 264,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "p99 {{scope}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.5, sum by (le, scope) (rate(SeaweedFS_filer_subscribe_hold_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])))",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "p50 {{scope}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Delivery Claim / Peer Low-Watermark Age",
|
||||
"description": "Seconds the filer's delivery claim and the peers' delivery and flush low-watermarks trail the clock; -1 while a tracked peer has not signalled",
|
||||
"type": "timeseries",
|
||||
"id": 265,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance) (SeaweedFS_filer_meta_delivery_claim_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "claim {{instance}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (instance, kind) (SeaweedFS_filer_meta_peer_low_watermark_age_seconds{cluster=~\"$cluster\"})",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"legendFormat": "peer low {{kind}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"title": "Metadata Subscribers Resumed from Disk",
|
||||
"description": "Times a metadata subscriber fell off the in-memory ring and resumed from the persisted log",
|
||||
"type": "timeseries",
|
||||
"id": 266,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (instance, scope) (rate(SeaweedFS_filer_subscribe_resume_from_disk{cluster=~\"$cluster\"}[$__rate_interval]))",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"legendFormat": "{{scope}} {{instance}}"
|
||||
}
|
||||
],
|
||||
"pluginVersion": "10.3.1"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -7262,7 +6642,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 87
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -7358,7 +6738,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 87
|
||||
"y": 71
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -7454,7 +6834,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 95
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -7550,7 +6930,7 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 95
|
||||
"y": 79
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
|
||||
Generated
-6
@@ -7143,10 +7143,8 @@ dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -7160,7 +7158,6 @@ dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"http",
|
||||
"hyper-util",
|
||||
"jsonb",
|
||||
"jsonwebtoken",
|
||||
"lance",
|
||||
"lance-datafusion",
|
||||
@@ -7193,7 +7190,6 @@ dependencies = [
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"x509-parser",
|
||||
"zstd",
|
||||
]
|
||||
@@ -7212,7 +7208,6 @@ dependencies = [
|
||||
"lance",
|
||||
"lance-index",
|
||||
"lance-linalg",
|
||||
"lance-select",
|
||||
"lance-table",
|
||||
"object_store",
|
||||
"roaring",
|
||||
@@ -7241,7 +7236,6 @@ dependencies = [
|
||||
"futures",
|
||||
"lance",
|
||||
"lance-arrow",
|
||||
"lance-index",
|
||||
"lance-linalg",
|
||||
"moka",
|
||||
"object_store",
|
||||
|
||||
@@ -22,7 +22,6 @@ object_store = "0.13"
|
||||
lance-index = "10"
|
||||
lance-linalg = "10"
|
||||
lance-table = "10"
|
||||
lance-select = "10"
|
||||
arrow-array = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-schema = "58"
|
||||
|
||||
@@ -139,7 +139,6 @@ mod tests {
|
||||
nprobes: Some(4),
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -168,7 +167,6 @@ mod tests {
|
||||
nprobes: Some(4),
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let filtered = ann(&dataset, &table, &query).await.unwrap();
|
||||
let exact = flat(&dataset, &table, &query).await.unwrap();
|
||||
@@ -206,7 +204,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
//! compares the elements as typed JSON values: Lance's compares their text, so
|
||||
//! 1 would match "1".
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::builder::{BooleanBuilder, StringBuilder};
|
||||
@@ -20,21 +19,13 @@ use lance::deps::datafusion::logical_expr::{
|
||||
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
|
||||
};
|
||||
|
||||
/// Set `filter` on the scanner with the JSONPath functions swapped and the
|
||||
/// conjuncts on `ready` indexed keys answered by their columns, which it
|
||||
/// returns.
|
||||
pub(crate) fn apply(
|
||||
scanner: &mut Scanner,
|
||||
filter: &str,
|
||||
ready: &HashMap<String, String>,
|
||||
) -> anyhow::Result<Vec<crate::indexed_filter::Indexed>> {
|
||||
/// Set `filter` on the scanner with the JSONPath functions swapped.
|
||||
pub(crate) fn apply(scanner: &mut Scanner, filter: &str) -> anyhow::Result<()> {
|
||||
scanner.filter(filter)?;
|
||||
let Some(expr) = scanner.get_expr_filter()? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let (expr, indexed) = crate::indexed_filter::rewrite(expr, ready);
|
||||
scanner.filter_expr(swap(expr)?);
|
||||
Ok(indexed)
|
||||
if let Some(expr) = scanner.get_expr_filter()? {
|
||||
scanner.filter_expr(swap(expr)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn swap(expr: Expr) -> Result<Expr> {
|
||||
@@ -277,7 +268,7 @@ mod tests {
|
||||
async fn keys(dataset: &Dataset, filter: &str, swapped: bool) -> anyhow::Result<Vec<String>> {
|
||||
let mut scanner = dataset.scan();
|
||||
if swapped {
|
||||
apply(&mut scanner, filter, &HashMap::new())?;
|
||||
apply(&mut scanner, filter)?;
|
||||
} else {
|
||||
scanner.filter(filter)?;
|
||||
}
|
||||
@@ -404,7 +395,6 @@ mod tests {
|
||||
apply(
|
||||
&mut scanner,
|
||||
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#,
|
||||
&HashMap::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut swapped = Vec::new();
|
||||
|
||||
@@ -104,7 +104,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,83 +198,6 @@ mod tests {
|
||||
assert_eq!(filtered.len(), 10);
|
||||
}
|
||||
|
||||
/// A filtered query through the index returns every eligible row when
|
||||
/// fewer than k match, though the probed partitions cannot hold them all:
|
||||
/// 400 of 4000 rows match, the index has 63 partitions and probes at most
|
||||
/// 32 of them.
|
||||
#[tokio::test]
|
||||
async fn filtered_ann_returns_every_match_below_k() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
|
||||
let tags: Vec<&str> = (0..4000)
|
||||
.map(|i| if i % 10 == 0 { "rare" } else { "common" })
|
||||
.collect();
|
||||
let mut dataset = testing::write_dataset(&dir, &rows, &tags).await;
|
||||
let table = VectorTable::resolve(&dataset, None, None).unwrap();
|
||||
build(
|
||||
&mut dataset,
|
||||
&table.vector_column,
|
||||
INDEX_NAME,
|
||||
partitions_for(4000),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(partitions_for(4000), 63);
|
||||
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
|
||||
let q = Query {
|
||||
filter: Some("tag = 'rare'".to_string()),
|
||||
..exact(vector, 500)
|
||||
};
|
||||
let flat = query::flat(&dataset, &table, &q).await.unwrap();
|
||||
assert_eq!(flat.len(), 400);
|
||||
let ann = query::ann(&dataset, &table, &q).await.unwrap();
|
||||
assert_eq!(ann.len(), 400, "the index path dropped eligible rows");
|
||||
for (f, a) in flat.iter().zip(ann.iter()) {
|
||||
assert!((f.distance - a.distance).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// When no more than k rows match, every match comes back with its own
|
||||
/// distance, nearest first — not the rows the probes missed with an
|
||||
/// infinite distance. Ten fragments of 400 rows, 5 of them matching.
|
||||
#[tokio::test]
|
||||
async fn filtered_ann_scores_every_match_at_or_below_k() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let rows = scattered(4000, 8, 0x9E3779B97F4A7C15);
|
||||
let tags: Vec<&str> = (0..4000)
|
||||
.map(|i| if i % 800 == 0 { "rare" } else { "common" })
|
||||
.collect();
|
||||
let mut dataset = testing::write_dataset(&dir, &rows[..400], &tags[..400]).await;
|
||||
for chunk in 1..10 {
|
||||
let r = chunk * 400..(chunk + 1) * 400;
|
||||
testing::append_dataset(&mut dataset, &rows[r.clone()], &tags[r]).await;
|
||||
}
|
||||
let table = VectorTable::resolve(&dataset, None, None).unwrap();
|
||||
build(&mut dataset, &table.vector_column, INDEX_NAME, 63, false)
|
||||
.await
|
||||
.unwrap();
|
||||
for vector in &scattered(3, 8, 0xD1B54A32D192ED03) {
|
||||
for k in [5, 10] {
|
||||
let q = Query {
|
||||
filter: Some("tag = 'rare'".to_string()),
|
||||
..exact(vector, k)
|
||||
};
|
||||
let flat = query::flat(&dataset, &table, &q).await.unwrap();
|
||||
assert_eq!(flat.len(), 5);
|
||||
let ann = query::ann(&dataset, &table, &q).await.unwrap();
|
||||
assert_eq!(ann.len(), 5);
|
||||
for (f, a) in flat.iter().zip(ann.iter()) {
|
||||
assert!(
|
||||
(f.distance - a.distance).abs() < 1e-5,
|
||||
"k={k}: {flat:?} vs {ann:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_probes_take_the_largest_index() {
|
||||
use lance::index::DatasetIndexExt;
|
||||
|
||||
@@ -1,739 +0,0 @@
|
||||
//! Filters on indexed metadata keys (see `weed_lance_worker::metadata_index`).
|
||||
//!
|
||||
//! The gateway renders a string `$eq` or `$in` on a key as an OR of
|
||||
//! `json_extract(metadata, P) = '"v"'` (or `IN`) and
|
||||
//! `json_array_contains(metadata, P, '"v"')` over the same values. In the
|
||||
//! parsed filter, such a top-level conjunct on a ready key is replaced by
|
||||
//! `array_has` / `array_has_any` on the key's column, which its LABEL_LIST
|
||||
//! index answers. Everything else, including `$ne` and `$nin`, stays on the
|
||||
//! JSON path.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use anyhow::Result;
|
||||
use jsonb::OwnedJsonb;
|
||||
use lance::Dataset;
|
||||
use lance::deps::datafusion::common::{Column, ScalarValue};
|
||||
use lance::deps::datafusion::functions_nested::expr_fn::{array_has, array_has_any};
|
||||
use lance::deps::datafusion::logical_expr::utils::split_conjunction_owned;
|
||||
use lance::deps::datafusion::logical_expr::{BinaryExpr, Expr, Operator};
|
||||
use lance::deps::datafusion::prelude::lit;
|
||||
use weed_lance_worker::metadata_index::{self, State};
|
||||
|
||||
const METADATA_COLUMN: &str = "metadata";
|
||||
|
||||
/// The column of each ready indexed key of `dataset`, by metadata key.
|
||||
pub(crate) fn ready_columns(dataset: &Dataset) -> HashMap<String, String> {
|
||||
let schema: arrow_schema::Schema = dataset.schema().into();
|
||||
metadata_index::indexed_keys(&schema)
|
||||
.into_iter()
|
||||
.filter(|k| k.state == Some(State::Ready))
|
||||
.map(|k| (k.key, k.column))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A conjunct a key's column answers: rows whose `column` holds one of
|
||||
/// `values`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct Indexed {
|
||||
pub column: String,
|
||||
pub values: Vec<String>,
|
||||
pub expr: Expr,
|
||||
}
|
||||
|
||||
/// `filter` with each top-level conjunct that is a string `$eq` or `$in` on
|
||||
/// a key in `ready` answered by the key's column instead, and those
|
||||
/// replacements.
|
||||
pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, Vec<Indexed>) {
|
||||
if ready.is_empty() {
|
||||
return (filter, Vec::new());
|
||||
}
|
||||
let mut indexed: Vec<Indexed> = Vec::new();
|
||||
let mut rewritten: Option<Expr> = None;
|
||||
for conjunct in split_conjunction_owned(filter) {
|
||||
let replacement = string_membership(&conjunct).and_then(|(key, values)| {
|
||||
let name = ready.get(&key)?;
|
||||
let column = Expr::Column(Column::from_name(name));
|
||||
let expr = match values.as_slice() {
|
||||
[one] => array_has(column, lit(one.as_str())),
|
||||
_ => {
|
||||
let values: Vec<ScalarValue> = values
|
||||
.iter()
|
||||
.map(|v| ScalarValue::Utf8(Some(v.clone())))
|
||||
.collect();
|
||||
let list = ScalarValue::new_list(&values, &arrow_schema::DataType::Utf8, true);
|
||||
array_has_any(column, Expr::Literal(ScalarValue::List(list), None))
|
||||
}
|
||||
};
|
||||
Some(Indexed {
|
||||
column: name.clone(),
|
||||
values,
|
||||
expr,
|
||||
})
|
||||
});
|
||||
let part = match replacement {
|
||||
Some(found) => {
|
||||
let expr = found.expr.clone();
|
||||
indexed.push(found);
|
||||
expr
|
||||
}
|
||||
None => conjunct,
|
||||
};
|
||||
rewritten = Some(and(rewritten, part));
|
||||
}
|
||||
(rewritten.unwrap_or(lit(true)), indexed)
|
||||
}
|
||||
|
||||
fn and(left: Option<Expr>, right: Expr) -> Expr {
|
||||
match left {
|
||||
Some(left) => Expr::BinaryExpr(BinaryExpr::new(
|
||||
Box::new(left),
|
||||
Operator::And,
|
||||
Box::new(right),
|
||||
)),
|
||||
None => right,
|
||||
}
|
||||
}
|
||||
|
||||
/// The key and string values of a conjunct meaning "the key holds one of
|
||||
/// these strings, or an array with one of them": the OR of scalar matches
|
||||
/// and array-member matches over the same values and key.
|
||||
fn string_membership(expr: &Expr) -> Option<(String, Vec<String>)> {
|
||||
let mut disjuncts = Vec::new();
|
||||
split_or(expr, &mut disjuncts);
|
||||
let mut path: Option<&str> = None;
|
||||
let mut scalars = BTreeSet::new();
|
||||
let mut elements = BTreeSet::new();
|
||||
for disjunct in disjuncts {
|
||||
let (p, kind) = match unwrap_false_coalesce(disjunct) {
|
||||
Expr::BinaryExpr(BinaryExpr {
|
||||
left,
|
||||
op: Operator::Eq,
|
||||
right,
|
||||
}) => (
|
||||
json_function(left, "json_extract")?,
|
||||
Member::Scalar(vec![utf8(right)?]),
|
||||
),
|
||||
Expr::InList(list) if !list.negated => (
|
||||
json_function(&list.expr, "json_extract")?,
|
||||
Member::Scalar(list.list.iter().map(utf8).collect::<Option<_>>()?),
|
||||
),
|
||||
Expr::ScalarFunction(f) if f.func.name() == "json_array_contains" => {
|
||||
let [data, p, value] = f.args.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
if !is_metadata(data) {
|
||||
return None;
|
||||
}
|
||||
let element: serde_json::Value = serde_json::from_str(utf8(value)?).ok()?;
|
||||
(utf8(p)?, Member::Element(element.as_str()?.to_string()))
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
if path.is_some_and(|q| q != p) {
|
||||
return None;
|
||||
}
|
||||
path = Some(p);
|
||||
match kind {
|
||||
Member::Scalar(values) => scalars.extend(values.into_iter().map(str::to_string)),
|
||||
Member::Element(value) => {
|
||||
elements.insert(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
let key = json_path_key(path?)?;
|
||||
let rendered: BTreeSet<String> = elements
|
||||
.iter()
|
||||
.map(|s| jsonb_text(s))
|
||||
.collect::<Option<_>>()?;
|
||||
(!elements.is_empty() && rendered == scalars).then(|| (key, elements.into_iter().collect()))
|
||||
}
|
||||
|
||||
enum Member<'a> {
|
||||
Scalar(Vec<&'a str>),
|
||||
Element(String),
|
||||
}
|
||||
|
||||
fn split_or<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) {
|
||||
match expr {
|
||||
Expr::BinaryExpr(BinaryExpr {
|
||||
left,
|
||||
op: Operator::Or,
|
||||
right,
|
||||
}) => {
|
||||
split_or(left, out);
|
||||
split_or(right, out);
|
||||
}
|
||||
Expr::Alias(alias) => split_or(&alias.expr, out),
|
||||
other => out.push(other),
|
||||
}
|
||||
}
|
||||
|
||||
/// `x` for `COALESCE(x, false)` or its simplified `x IS NOT NULL AND x`;
|
||||
/// either is false where `x` is null, which a top-level conjunct treats alike.
|
||||
fn unwrap_false_coalesce(expr: &Expr) -> &Expr {
|
||||
match expr {
|
||||
Expr::ScalarFunction(f) if f.func.name() == "coalesce" => match f.args.as_slice() {
|
||||
[x, Expr::Literal(ScalarValue::Boolean(Some(false)), _)] => x,
|
||||
_ => expr,
|
||||
},
|
||||
Expr::BinaryExpr(BinaryExpr {
|
||||
left,
|
||||
op: Operator::And,
|
||||
right,
|
||||
}) => match left.as_ref() {
|
||||
Expr::IsNotNull(x) if x == right => right,
|
||||
_ => expr,
|
||||
},
|
||||
_ => expr,
|
||||
}
|
||||
}
|
||||
|
||||
fn utf8(expr: &Expr) -> Option<&str> {
|
||||
match expr {
|
||||
Expr::Literal(ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)), _) => {
|
||||
Some(s.as_str())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_metadata(expr: &Expr) -> bool {
|
||||
matches!(expr, Expr::Column(c) if c.name == METADATA_COLUMN)
|
||||
}
|
||||
|
||||
/// The JSONPath literal of `name(metadata, path)`.
|
||||
fn json_function<'a>(expr: &'a Expr, name: &str) -> Option<&'a str> {
|
||||
match expr {
|
||||
Expr::ScalarFunction(f) if f.func.name() == name => match f.args.as_slice() {
|
||||
[data, path] if is_metadata(data) => utf8(path),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The key of a `$["key"]` path, with `\\` and `\"` unescaped.
|
||||
fn json_path_key(path: &str) -> Option<String> {
|
||||
let inner = path.strip_prefix("$[\"")?.strip_suffix("\"]")?;
|
||||
let mut key = String::with_capacity(inner.len());
|
||||
let mut chars = inner.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
match c {
|
||||
'\\' => match chars.next()? {
|
||||
c @ ('\\' | '"') => key.push(c),
|
||||
_ => return None,
|
||||
},
|
||||
'"' => return None,
|
||||
c => key.push(c),
|
||||
}
|
||||
}
|
||||
Some(key)
|
||||
}
|
||||
|
||||
/// What json_extract returns for a stored string `s`.
|
||||
fn jsonb_text(s: &str) -> Option<String> {
|
||||
let value: OwnedJsonb = serde_json::to_string(s).ok()?.parse().ok()?;
|
||||
Some(value.to_string())
|
||||
}
|
||||
|
||||
/// Segments searched at once when counting.
|
||||
const PARALLEL_SEGMENTS: usize = 8;
|
||||
|
||||
/// The rows of `scope` (every fragment when None) whose column matches
|
||||
/// `indexed`, or None once there are more than `limit`. Segments are
|
||||
/// searched newest first, several at once, and counting stops as soon as
|
||||
/// the total passes the limit. Rows the index has not reached are counted
|
||||
/// by a read of the column. Deleted rows an index segment still lists
|
||||
/// count too: the total is an upper bound, which only ever sends a query
|
||||
/// to the vector index instead of an exact scan.
|
||||
pub(crate) async fn count_indexed(
|
||||
dataset: &Dataset,
|
||||
indexed: &Indexed,
|
||||
scope: Option<&[u64]>,
|
||||
limit: u64,
|
||||
) -> Result<Option<u64>> {
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use lance::index::DatasetIndexInternalExt;
|
||||
use lance_index::metrics::NoOpMetricsCollector;
|
||||
use lance_index::scalar::{LabelListQuery, SearchResult};
|
||||
use lance_select::RowSetOps;
|
||||
|
||||
let scope: Option<roaring::RoaringBitmap> =
|
||||
scope.map(|ids| ids.iter().map(|id| *id as u32).collect());
|
||||
let segments = metadata_index::segments(dataset, &indexed.column).await?;
|
||||
let query = LabelListQuery::HasAnyLabel(
|
||||
indexed
|
||||
.values
|
||||
.iter()
|
||||
.map(|v| ScalarValue::Utf8(Some(v.clone())))
|
||||
.collect(),
|
||||
);
|
||||
let mut searches: Vec<futures::future::BoxFuture<'_, Result<u64>>> = Vec::new();
|
||||
for segment in segments.iter().rev() {
|
||||
let fragments = match &scope {
|
||||
Some(scope) => &segment.fragments & scope,
|
||||
None => segment.fragments.clone(),
|
||||
};
|
||||
if fragments.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let uuid = segment.uuid;
|
||||
let query = &query;
|
||||
searches.push(Box::pin(async move {
|
||||
let index = dataset
|
||||
.open_scalar_index(&indexed.column, &uuid, &NoOpMetricsCollector)
|
||||
.await?;
|
||||
let mut found = match index.search(query, &NoOpMetricsCollector).await? {
|
||||
SearchResult::Exact(rows) | SearchResult::AtMost(rows) => rows.true_rows(),
|
||||
SearchResult::AtLeast(_) => anyhow::bail!("a LABEL_LIST index answered AtLeast"),
|
||||
};
|
||||
found.retain_fragments(fragments.iter());
|
||||
Ok(found.len().unwrap_or(u64::MAX))
|
||||
}));
|
||||
}
|
||||
let mut counts = futures::stream::iter(searches).buffered(PARALLEL_SEGMENTS);
|
||||
let mut total = 0u64;
|
||||
while let Some(count) = counts.try_next().await? {
|
||||
total = total.saturating_add(count);
|
||||
if total > limit {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let unindexed: Vec<lance_table::format::Fragment> = dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.filter(|f| !segments.iter().any(|s| s.fragments.contains(f.id as u32)))
|
||||
.filter(|f| scope.as_ref().is_none_or(|s| s.contains(f.id as u32)))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !unindexed.is_empty() {
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.filter_expr(indexed.expr.clone())
|
||||
.with_fragments(unindexed);
|
||||
total = total.saturating_add(scanner.count_rows().await?);
|
||||
}
|
||||
Ok((total <= limit).then_some(total))
|
||||
}
|
||||
|
||||
/// How a filter compares a key.
|
||||
pub mod operand {
|
||||
pub const STRING: u8 = 1;
|
||||
pub const NUMBER: u8 = 2;
|
||||
pub const BOOL: u8 = 4;
|
||||
pub const EXISTS: u8 = 8;
|
||||
}
|
||||
|
||||
/// The metadata keys `filter` reads, each with the [`operand`] kinds it is
|
||||
/// compared with.
|
||||
pub fn filter_keys(dataset: &Dataset, filter: &str) -> Result<BTreeMap<String, u8>> {
|
||||
let mut scanner = dataset.scan();
|
||||
scanner.filter(filter)?;
|
||||
let mut keys = BTreeMap::new();
|
||||
if let Some(expr) = scanner.get_expr_filter()? {
|
||||
collect_keys(&expr, &mut keys);
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
fn literal_kind(text: &str) -> u8 {
|
||||
match serde_json::from_str::<serde_json::Value>(text) {
|
||||
Ok(serde_json::Value::String(_)) => operand::STRING,
|
||||
Ok(serde_json::Value::Number(_)) => operand::NUMBER,
|
||||
Ok(serde_json::Value::Bool(_)) => operand::BOOL,
|
||||
// json_extract's text of a string with control characters is not
|
||||
// JSON, but it is quoted.
|
||||
_ if text.starts_with('"') => operand::STRING,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_keys(expr: &Expr, keys: &mut BTreeMap<String, u8>) {
|
||||
let mut note = |path: &str, kind: u8| {
|
||||
if let Some(key) = json_path_key(path) {
|
||||
*keys.entry(key).or_default() |= kind;
|
||||
}
|
||||
};
|
||||
match expr {
|
||||
Expr::TryCast(cast) if extract(&cast.expr).is_some() => {
|
||||
note(extract(&cast.expr).unwrap_or_default(), operand::NUMBER);
|
||||
return;
|
||||
}
|
||||
Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
|
||||
if let (Some(path), Some(value)) = (extract(left), utf8(right)) {
|
||||
note(path, literal_kind(value));
|
||||
return;
|
||||
}
|
||||
}
|
||||
Expr::InList(list) => {
|
||||
if let Some(path) = extract(&list.expr) {
|
||||
let kind = list
|
||||
.list
|
||||
.iter()
|
||||
.filter_map(utf8)
|
||||
.fold(0, |k, v| k | literal_kind(v));
|
||||
note(path, kind);
|
||||
return;
|
||||
}
|
||||
}
|
||||
Expr::ScalarFunction(f) if is_json_function(f.func.name()) => {
|
||||
if let [data, path, rest @ ..] = f.args.as_slice()
|
||||
&& is_metadata(data)
|
||||
&& let Some(path) = utf8(path)
|
||||
{
|
||||
let kind = match (f.func.name(), rest) {
|
||||
("json_exists", _) => operand::EXISTS,
|
||||
(_, [value]) => utf8(value).map_or(0, literal_kind),
|
||||
_ => 0,
|
||||
};
|
||||
note(path, kind);
|
||||
return;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
for child in children(expr) {
|
||||
collect_keys(child, keys);
|
||||
}
|
||||
}
|
||||
|
||||
fn extract(expr: &Expr) -> Option<&str> {
|
||||
json_function(expr, "json_extract")
|
||||
}
|
||||
|
||||
fn is_json_function(name: &str) -> bool {
|
||||
matches!(name, "json_extract" | "json_exists" | "json_array_contains")
|
||||
}
|
||||
|
||||
fn children(expr: &Expr) -> Vec<&Expr> {
|
||||
use lance::deps::datafusion::common::tree_node::TreeNode;
|
||||
let mut out = Vec::new();
|
||||
let _ = expr.apply_children(|child| {
|
||||
out.push(child);
|
||||
Ok(lance::deps::datafusion::common::tree_node::TreeNodeRecursion::Continue)
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::query::{self, Query};
|
||||
use crate::table::VectorTable;
|
||||
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance_index::IndexType;
|
||||
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
|
||||
use lance_linalg::distance::DistanceType;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const DIM: usize = 4;
|
||||
|
||||
/// The gateway's text for `{"author": "<v>"}`.
|
||||
fn eq(field: &str, json: &str) -> String {
|
||||
let json = json.replace('\'', "''");
|
||||
format!(
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"{field}\"]') = '{json}', false) OR COALESCE(json_array_contains(`metadata`, '$[\"{field}\"]', '{json}'), false))"
|
||||
)
|
||||
}
|
||||
|
||||
/// The gateway's text for `{"author": {"$in": [<strings>]}}`.
|
||||
fn among(values: &[&str]) -> String {
|
||||
let scalars: Vec<String> = values.iter().map(|v| format!("'\"{v}\"'")).collect();
|
||||
let mut groups = vec![format!(
|
||||
"COALESCE(json_extract(`metadata`, '$[\"author\"]') in ({}), false)",
|
||||
scalars.join(", ")
|
||||
)];
|
||||
for v in values {
|
||||
groups.push(format!(
|
||||
"COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"{v}\"'), false)"
|
||||
));
|
||||
}
|
||||
format!("({})", groups.join(" OR "))
|
||||
}
|
||||
|
||||
/// Rows `u0..u<n>` by author, plus a few others, in fragments of 100,
|
||||
/// with the author column ready and indexed and a vector index.
|
||||
async fn dataset(dir: &TempDir, rows: usize) -> Dataset {
|
||||
let metadata = Field::new("metadata", DataType::Utf8, true)
|
||||
.with_metadata([("ARROW:extension:name".to_string(), "arrow.json".to_string())].into());
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("key", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(
|
||||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||||
DIM as i32,
|
||||
),
|
||||
false,
|
||||
),
|
||||
metadata,
|
||||
metadata_index::field("_mk_0", "author", State::Ready),
|
||||
]));
|
||||
let metas: Vec<serde_json::Value> = (0..rows)
|
||||
.map(|i| match i % 10 {
|
||||
0 => serde_json::json!({"author": "rare"}),
|
||||
1 => serde_json::json!({"author": ["many", "rare2"]}),
|
||||
_ => serde_json::json!({"author": "many", "n": i}),
|
||||
})
|
||||
.collect();
|
||||
let batches: Vec<RecordBatch> = (0..rows)
|
||||
.step_by(100)
|
||||
.map(|start| {
|
||||
let range = start..(start + 100).min(rows);
|
||||
RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(StringArray::from(
|
||||
range.clone().map(|i| format!("k{i}")).collect::<Vec<_>>(),
|
||||
)),
|
||||
Arc::new(FixedSizeListArray::from_iter_primitive::<
|
||||
arrow_array::types::Float32Type,
|
||||
_,
|
||||
_,
|
||||
>(
|
||||
range.clone().map(|i| {
|
||||
Some((0..DIM).map(move |j| Some(((i * 31 + j * 17) % 97) as f32)))
|
||||
}),
|
||||
DIM as i32,
|
||||
)),
|
||||
Arc::new(StringArray::from(
|
||||
range
|
||||
.clone()
|
||||
.map(|i| Some(metas[i].to_string()))
|
||||
.collect::<Vec<_>>(),
|
||||
)),
|
||||
metadata_index::column_from_json(range.map(|i| Some(&metas[i])), "author"),
|
||||
],
|
||||
)
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
let uri = dir.path().to_str().unwrap();
|
||||
let mut dataset = None::<Dataset>;
|
||||
for batch in batches {
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
|
||||
dataset = Some(match dataset {
|
||||
None => Dataset::write(reader, uri, None).await.unwrap(),
|
||||
Some(mut d) => {
|
||||
d.append(reader, None).await.unwrap();
|
||||
d
|
||||
}
|
||||
});
|
||||
}
|
||||
let mut dataset = dataset.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["_mk_0"],
|
||||
IndexType::LabelList,
|
||||
None,
|
||||
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crate::index::build(&mut dataset, "vector", crate::index::INDEX_NAME, 4, false)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
}
|
||||
|
||||
fn rewritten(dataset: &Dataset, filter: &str) -> (String, Option<String>) {
|
||||
let mut scanner = dataset.scan();
|
||||
let indexed = crate::filter::apply(&mut scanner, filter, &ready_columns(dataset)).unwrap();
|
||||
(
|
||||
scanner.get_expr_filter().unwrap().unwrap().to_string(),
|
||||
indexed.first().map(|i| i.expr.to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn string_eq_and_in_on_a_ready_key_use_its_column() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dataset = dataset(&dir, 200).await;
|
||||
assert_eq!(
|
||||
ready_columns(&dataset),
|
||||
HashMap::from([("author".to_string(), "_mk_0".to_string())])
|
||||
);
|
||||
for (filter, want) in [
|
||||
(eq("author", "\"rare\""), Some("array_has(_mk_0, Utf8(\"rare\"))")),
|
||||
(
|
||||
among(&["rare", "many"]),
|
||||
Some("array_has_any(_mk_0, List([many, rare]))"),
|
||||
),
|
||||
(
|
||||
format!("{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3", eq("author", "\"rare\"")),
|
||||
Some("array_has(_mk_0, Utf8(\"rare\"))"),
|
||||
),
|
||||
(eq("author", "\"o'neil \\\"q\\\"\""), Some("array_has(_mk_0, Utf8(\"o'neil \"q\"\"))")),
|
||||
(
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u0001d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
|
||||
Some("array_has(_mk_0, Utf8(\"c\u{1}d\n\"))"),
|
||||
),
|
||||
// The scalar half's text is not what json_extract returns for
|
||||
// the array half's string, so the two do not agree.
|
||||
(
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"c\\u{1}d\\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"c\\u0001d\\n\"'), false))".to_string(),
|
||||
None,
|
||||
),
|
||||
// Not a string, not this key, or not every value on both sides.
|
||||
(eq("author", "true"), None),
|
||||
(eq("other", "\"rare\""), None),
|
||||
(format!("NOT {}", eq("author", "\"rare\"")), None),
|
||||
(format!("NOT {}", among(&["rare", "many"])), None),
|
||||
(
|
||||
"COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"rare\"', false)".to_string(),
|
||||
None,
|
||||
),
|
||||
(
|
||||
among(&["rare", "many"]).replace("'\"many\"'), false))", "'\"other\"'), false))"),
|
||||
None,
|
||||
),
|
||||
("json_exists(`metadata`, '$[\"author\"]')".to_string(), None),
|
||||
(format!("({} OR json_exists(`metadata`, '$[\"n\"]'))", eq("author", "\"rare\"")), None),
|
||||
] {
|
||||
let (expr, indexed) = rewritten(&dataset, &filter);
|
||||
assert_eq!(indexed.as_deref(), want, "{filter}");
|
||||
assert_eq!(expr.contains("json_"), want.is_none() || filter.contains(" AND "), "{filter}: {expr}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The planner scores few matches exactly and leaves many to the vector
|
||||
/// index, per scope.
|
||||
#[tokio::test]
|
||||
async fn few_indexed_matches_are_scored_exactly() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dataset = dataset(&dir, 1000).await;
|
||||
let table = VectorTable::resolve(&dataset, None, None).unwrap();
|
||||
let query = |filter: &str, fragments: Option<Vec<u64>>| Query {
|
||||
vector: vec![1.0, 2.0, 3.0, 4.0],
|
||||
k: 10,
|
||||
filter: Some(filter.to_string()),
|
||||
postfilter: false,
|
||||
distance: DistanceType::L2,
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let rare = eq("author", "\"rare\"");
|
||||
let many = eq("author", "\"many\"");
|
||||
let few = |q: Query| {
|
||||
let (dataset, table) = (&dataset, &table);
|
||||
async move {
|
||||
query::few_indexed_matches(dataset, table, &q)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
// 100 and 900 matches, and 90 in the first two fragments.
|
||||
assert!(few(query(&rare, None)).await);
|
||||
assert!(!few(query(&many, None)).await);
|
||||
let first = Some(vec![0, 1]);
|
||||
assert!(few(query(&many, first.clone())).await);
|
||||
assert!(
|
||||
!few(Query {
|
||||
exact_match_limit: Some(0),
|
||||
..query(&rare, None)
|
||||
})
|
||||
.await
|
||||
);
|
||||
// A JSON-only filter is not counted.
|
||||
assert!(!few(query("json_exists(`metadata`, '$[\"n\"]')", None)).await);
|
||||
for (filter, fragments) in [(&rare, None), (&many, first)] {
|
||||
let q = query(filter, fragments);
|
||||
let exact = query::flat(&dataset, &table, &q).await.unwrap();
|
||||
let got = query::ann(&dataset, &table, &q).await.unwrap();
|
||||
assert_eq!(got.len(), 10);
|
||||
for (g, e) in got.iter().zip(&exact) {
|
||||
assert!(
|
||||
(g.distance - e.distance).abs() < 1e-4,
|
||||
"{got:?} vs {exact:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_plan_reads_the_scalar_index() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dataset = dataset(&dir, 300).await;
|
||||
let mut scanner = dataset.scan();
|
||||
crate::filter::apply(
|
||||
&mut scanner,
|
||||
&eq("author", "\"rare\""),
|
||||
&ready_columns(&dataset),
|
||||
)
|
||||
.unwrap();
|
||||
let plan = scanner.explain_plan(true).await.unwrap();
|
||||
assert!(plan.contains("LabelList"), "{plan}");
|
||||
assert!(!plan.contains("json_"), "{plan}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn filter_keys_note_how_each_key_is_compared() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dataset = dataset(&dir, 100).await;
|
||||
let filter = format!(
|
||||
"{} AND TRY_CAST(json_extract(`metadata`, '$[\"n\"]') AS DOUBLE) > 3 AND NOT json_exists(`metadata`, '$[\"a.b\\\"c\"]') AND COALESCE(json_extract(`metadata`, '$[\"f\"]') = 'true', false)",
|
||||
eq("author", "\"rare\"")
|
||||
);
|
||||
let keys = filter_keys(&dataset, &filter).unwrap();
|
||||
assert_eq!(
|
||||
keys,
|
||||
BTreeMap::from([
|
||||
("a.b\"c".to_string(), operand::EXISTS),
|
||||
("author".to_string(), operand::STRING),
|
||||
("f".to_string(), operand::BOOL),
|
||||
("n".to_string(), operand::NUMBER),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/// Counting walks the segments, keeps to the scope and stops once over
|
||||
/// the limit.
|
||||
#[tokio::test]
|
||||
async fn counts_follow_segments_and_scope() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dataset = dataset(&dir, 1000).await;
|
||||
dataset.drop_index("_mk_0_idx").await.unwrap();
|
||||
// Ten fragments of 100 rows: four segments of up to 300 rows.
|
||||
let ids: Vec<u64> = dataset.fragments().iter().map(|f| f.id).collect();
|
||||
metadata_index::index_fragments(&mut dataset, "_mk_0", 300, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let segments = metadata_index::segments(&dataset, "_mk_0").await.unwrap();
|
||||
assert_eq!(segments.len(), 4);
|
||||
let ready = ready_columns(&dataset);
|
||||
let indexed =
|
||||
crate::filter::apply(&mut dataset.scan(), &eq("author", "\"rare\""), &ready).unwrap();
|
||||
let rare = &indexed[0];
|
||||
// "rare" in rows i % 10 == 0, "rare2" never matches it.
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, None, 1_000).await.unwrap(),
|
||||
Some(100)
|
||||
);
|
||||
assert_eq!(count_indexed(&dataset, rare, None, 99).await.unwrap(), None);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&ids[..2]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(20)
|
||||
);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&ids[9..]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(10)
|
||||
);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&[]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(0)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@
|
||||
pub mod coverage;
|
||||
mod filter;
|
||||
pub mod index;
|
||||
pub mod indexed_filter;
|
||||
pub mod maintenance;
|
||||
pub mod query;
|
||||
pub mod segments;
|
||||
|
||||
@@ -309,7 +309,6 @@ async fn main() -> Result<()> {
|
||||
nprobes,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let hits = if flat {
|
||||
query::flat(&dataset, &vector_table, &query).await?
|
||||
|
||||
@@ -312,7 +312,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let mut ids: Vec<u64> = query::flat(&pinned, &resolved, &q)
|
||||
.await
|
||||
@@ -735,7 +734,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
assert!(query::flat(&dataset, &resolved, &q).await.is_err());
|
||||
}
|
||||
|
||||
@@ -39,15 +39,8 @@ pub struct Query {
|
||||
/// a worker the unindexed-fragment sweep. Listed ids that no longer exist
|
||||
/// scan nothing.
|
||||
pub fragments: Option<Vec<u64>>,
|
||||
/// Score the matches exactly instead of searching the vector index when
|
||||
/// the filter's indexed metadata keys match at most this many rows of the
|
||||
/// scope; 0 never does. None takes [`DEFAULT_EXACT_MATCH_LIMIT`].
|
||||
pub exact_match_limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Up to this many matches of an indexed metadata key are scored exactly.
|
||||
pub const DEFAULT_EXACT_MATCH_LIMIT: usize = 256;
|
||||
|
||||
/// One row of a top-K answer.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct Hit {
|
||||
@@ -60,101 +53,14 @@ pub struct Hit {
|
||||
|
||||
/// Exact top-K over every fragment, ignoring any index.
|
||||
pub async fn flat(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
|
||||
run(scanner(dataset, table, query, false).await?.0, table).await
|
||||
run(scanner(dataset, table, query, false).await?, table).await
|
||||
}
|
||||
|
||||
/// Top-K with the ANN index where it covers the fragment and a flat pass over
|
||||
/// the rest. Whether the index is used is Lance's decision per fragment, so
|
||||
/// rows written after the last index build are always candidates.
|
||||
///
|
||||
/// A prefiltered search sees only the probed partitions: it can come back
|
||||
/// short of k, or with unscored rows (infinite distance) when at most k rows
|
||||
/// match. Either way it is answered again by an exact scan of the same scope.
|
||||
///
|
||||
/// When the filter's indexed metadata keys match few rows of that scope,
|
||||
/// they are scored exactly to begin with. The decision is per scope, so
|
||||
/// exact answers of several scopes merge into the exact answer.
|
||||
pub async fn ann(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
|
||||
if few_indexed_matches(dataset, table, query).await? {
|
||||
return exact_in_scope(dataset, table, query).await;
|
||||
}
|
||||
let (scanner, indexed) = scanner(dataset, table, query, true).await?;
|
||||
let hits = run(scanner, table).await?;
|
||||
if indexed
|
||||
&& query.filter.is_some()
|
||||
&& !query.postfilter
|
||||
&& (hits.len() < query.k || hits.iter().any(|h| !h.distance.is_finite()))
|
||||
{
|
||||
return exact_in_scope(dataset, table, query).await;
|
||||
}
|
||||
Ok(hits)
|
||||
}
|
||||
|
||||
/// [`flat`] over what [`ann`] searches for `query`: the listed fragments plus
|
||||
/// the live coverage of the listed index segments, or the whole table.
|
||||
async fn exact_in_scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Vec<Hit>> {
|
||||
let exact = Query {
|
||||
index_segments: None,
|
||||
fragments: scope(dataset, table, query).await?,
|
||||
..query.clone()
|
||||
};
|
||||
run(scanner(dataset, table, &exact, false).await?.0, table).await
|
||||
}
|
||||
|
||||
/// The fragments [`ann`] searches for `query`, or None for the whole table.
|
||||
async fn scope(dataset: &Dataset, table: &VectorTable, query: &Query) -> Result<Option<Vec<u64>>> {
|
||||
Ok(match &query.index_segments {
|
||||
Some(segments) if !segments.is_empty() => {
|
||||
let mut ids = query.fragments.clone().unwrap_or_default();
|
||||
ids.extend(
|
||||
crate::segments::covered_fragments(dataset, &table.vector_column, segments).await?,
|
||||
);
|
||||
Some(ids)
|
||||
}
|
||||
_ => query.fragments.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the rows of [`ann`]'s scope matching the filter's indexed
|
||||
/// metadata keys, counted through their scalar indexes, are at most the
|
||||
/// query's limit. Only a search through the vector index asks.
|
||||
pub(crate) async fn few_indexed_matches(
|
||||
dataset: &Dataset,
|
||||
table: &VectorTable,
|
||||
query: &Query,
|
||||
) -> Result<bool> {
|
||||
let limit = query.exact_match_limit.unwrap_or(DEFAULT_EXACT_MATCH_LIMIT);
|
||||
let Some(filter) = &query.filter else {
|
||||
return Ok(false);
|
||||
};
|
||||
if limit == 0 || query.postfilter || query.index_segments.as_ref().is_some_and(|s| s.is_empty())
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let ready = crate::indexed_filter::ready_columns(dataset);
|
||||
if ready.is_empty()
|
||||
|| crate::index::index_names(dataset, &table.vector_column)
|
||||
.await?
|
||||
.is_empty()
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let indexed = crate::filter::apply(&mut dataset.scan(), filter, &ready)?;
|
||||
if indexed.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let scope = scope(dataset, table, query).await?;
|
||||
// An AND matches at most what its most selective indexed conjunct does.
|
||||
for conjunct in &indexed {
|
||||
let count =
|
||||
crate::indexed_filter::count_indexed(dataset, conjunct, scope.as_deref(), limit as u64)
|
||||
.await?;
|
||||
tracing::debug!(column = %conjunct.column, ?count, limit, "indexed metadata filter");
|
||||
if count.is_some() {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
run(scanner(dataset, table, query, true).await?, table).await
|
||||
}
|
||||
|
||||
/// Fetch rows by key, projected to `columns` (all of them when None).
|
||||
@@ -195,7 +101,7 @@ async fn scanner(
|
||||
table: &VectorTable,
|
||||
query: &Query,
|
||||
use_index: bool,
|
||||
) -> Result<(Scanner, bool)> {
|
||||
) -> Result<Scanner> {
|
||||
if query.vector.len() != table.dimension {
|
||||
bail!(
|
||||
"query dim({}) does not match the table's dim({})",
|
||||
@@ -211,24 +117,18 @@ async fn scanner(
|
||||
.use_index(use_index)
|
||||
.with_row_id();
|
||||
if let Some(filter) = &query.filter {
|
||||
let ready = crate::indexed_filter::ready_columns(dataset);
|
||||
crate::filter::apply(&mut scanner, filter, &ready)?;
|
||||
crate::filter::apply(&mut scanner, filter)?;
|
||||
scanner.prefilter(!query.postfilter);
|
||||
}
|
||||
// An explicitly empty segment list searches no index, so there is
|
||||
// nothing to size probes for.
|
||||
let no_segments = query.index_segments.as_ref().is_some_and(|s| s.is_empty());
|
||||
let mut indexed = false;
|
||||
if use_index && !no_segments {
|
||||
let probes = match query.nprobes {
|
||||
Some(n) => crate::index::index_names(dataset, &table.vector_column)
|
||||
.await?
|
||||
.first()
|
||||
.map(|_| crate::index::Probes { min: n, max: n }),
|
||||
Some(n) => Some(crate::index::Probes { min: n, max: n }),
|
||||
None => crate::index::default_probes(dataset, &table.vector_column).await?,
|
||||
};
|
||||
if let Some(probes) = probes {
|
||||
indexed = true;
|
||||
// Lance searches `min` partitions, and under a prefilter keeps
|
||||
// going up to `max` until it has k rows. An open bound would
|
||||
// make it prepare every partition for that, which costs as much
|
||||
@@ -278,7 +178,7 @@ async fn scanner(
|
||||
columns.push(DISTANCE_COLUMN.to_string());
|
||||
columns.push(ROW_ID_COLUMN.to_string());
|
||||
scanner.project(&columns)?;
|
||||
Ok((scanner, indexed))
|
||||
Ok(scanner)
|
||||
}
|
||||
|
||||
async fn run(scanner: Scanner, table: &VectorTable) -> Result<Vec<Hit>> {
|
||||
@@ -361,7 +261,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let hits = flat(&dataset, &table, &query).await.unwrap();
|
||||
assert_eq!(hits.len(), 2);
|
||||
@@ -403,7 +302,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -483,7 +381,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -587,7 +484,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
assert!(flat(&dataset, &table, &query).await.is_err());
|
||||
}
|
||||
@@ -611,7 +507,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let indexed = ann(&dataset, &table, &query).await.unwrap();
|
||||
let exact = flat(&dataset, &table, &query).await.unwrap();
|
||||
@@ -620,36 +515,4 @@ mod tests {
|
||||
exact.iter().map(|h| &h.key).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
/// An explicit probe count does not make an unindexed table's scan count
|
||||
/// as indexed, so a short filtered answer is not scanned a second time.
|
||||
#[tokio::test]
|
||||
async fn explicit_probes_need_an_index() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let rows = crate::testing::vectors(300, DIM);
|
||||
let mut dataset = write_dataset(&dir, &rows, &["a"; 300]).await;
|
||||
let table = VectorTable::resolve(&dataset, None, None).unwrap();
|
||||
let query = Query {
|
||||
vector: rows[0].clone(),
|
||||
k: 10,
|
||||
filter: Some("tag = 'a'".to_string()),
|
||||
postfilter: false,
|
||||
distance: DistanceType::L2,
|
||||
nprobes: Some(4),
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
assert!(!scanner(&dataset, &table, &query, true).await.unwrap().1);
|
||||
crate::index::build(
|
||||
&mut dataset,
|
||||
&table.vector_column,
|
||||
crate::index::INDEX_NAME,
|
||||
4,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(scanner(&dataset, &table, &query, true).await.unwrap().1);
|
||||
}
|
||||
}
|
||||
@@ -139,7 +139,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let want: Vec<(u64, u32)> = query::ann(&dataset, &resolved, &base)
|
||||
.await
|
||||
@@ -240,7 +239,6 @@ mod tests {
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: Some(vec![u64::MAX]),
|
||||
exact_match_limit: None,
|
||||
};
|
||||
assert!(
|
||||
query::ann(&dataset, &resolved, &q)
|
||||
|
||||
@@ -44,7 +44,6 @@ fn query_for(vector: Vec<f32>, k: usize) -> Query {
|
||||
nprobes: Some(4),
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,6 @@ tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
lance-index = "10"
|
||||
tempfile = "3"
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -1,392 +0,0 @@
|
||||
//! How often, and how slowly, each index's queries filter on each metadata
|
||||
//! key over the last hour. Keys come from the parsed filter; the number of
|
||||
//! indexes and of keys per index is bounded, so arbitrary keys cannot grow
|
||||
//! memory without limit. Not exported as metrics: a key is no label.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use weed_vector_core::indexed_filter::operand;
|
||||
|
||||
use crate::pb::vector as pb;
|
||||
|
||||
const BUCKETS: usize = 12;
|
||||
const BUCKET: Duration = Duration::from_secs(300);
|
||||
pub const WINDOW: Duration = Duration::from_secs(BUCKET.as_secs() * BUCKETS as u64);
|
||||
pub const MAX_INDEXES: usize = 512;
|
||||
pub const MAX_KEYS_PER_INDEX: usize = 32;
|
||||
const MAX_RETIRED_TABLES: usize = 4;
|
||||
|
||||
#[derive(Default, Clone, Copy)]
|
||||
struct Bucket {
|
||||
/// The bucket's number since the stats started.
|
||||
at: u64,
|
||||
queries: u64,
|
||||
seconds: f64,
|
||||
/// The operand kinds the bucket's queries compared the key with.
|
||||
kinds: u8,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct KeyStats {
|
||||
buckets: [Bucket; BUCKETS],
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct IndexStats {
|
||||
/// The table the stats were counted on: an index dropped and created
|
||||
/// again under its name is another table.
|
||||
table: String,
|
||||
/// Tables the name held before, whose late queries are not counted.
|
||||
retired: Vec<String>,
|
||||
keys: HashMap<String, KeyStats>,
|
||||
/// The last bucket a key was left uncounted in.
|
||||
truncated: Option<u64>,
|
||||
last: u64,
|
||||
}
|
||||
|
||||
impl IndexStats {
|
||||
/// Starts counting on `table`, retiring the table counted on before.
|
||||
fn replace(&mut self, table: &str) {
|
||||
let mut retired = std::mem::take(&mut self.retired);
|
||||
if !self.table.is_empty() {
|
||||
retired.push(std::mem::take(&mut self.table));
|
||||
if retired.len() > MAX_RETIRED_TABLES {
|
||||
retired.remove(0);
|
||||
}
|
||||
}
|
||||
*self = IndexStats {
|
||||
table: table.to_string(),
|
||||
retired,
|
||||
last: self.last,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether bucket `at` is still in the window of bucket `now`.
|
||||
fn live(at: u64, now: u64) -> bool {
|
||||
at + BUCKETS as u64 > now
|
||||
}
|
||||
|
||||
pub struct FilterStats {
|
||||
start: Instant,
|
||||
indexes: Mutex<HashMap<String, IndexStats>>,
|
||||
}
|
||||
|
||||
impl Default for FilterStats {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FilterStats {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
start: Instant::now(),
|
||||
indexes: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket(&self, now: Instant) -> u64 {
|
||||
(now.duration_since(self.start).as_secs() / BUCKET.as_secs()) + 1
|
||||
}
|
||||
|
||||
/// Drops the stats of `index`, which now holds `table` ("" once
|
||||
/// dropped): late queries of the table it held before are not counted.
|
||||
pub fn forget(&self, index: &str, table: &str) {
|
||||
let mut indexes = self.indexes.lock().unwrap();
|
||||
if let Some(stats) = indexes.get_mut(index)
|
||||
&& (table.is_empty() || stats.table != table)
|
||||
{
|
||||
stats.replace(table);
|
||||
}
|
||||
}
|
||||
|
||||
/// One query of `index`, whose table is `table`, that took `elapsed` and
|
||||
/// filtered on `keys`. Stats counted on another table under the name are
|
||||
/// dropped; a query of a table the name no longer holds is not counted.
|
||||
pub fn record(&self, index: &str, table: &str, keys: &BTreeMap<String, u8>, elapsed: Duration) {
|
||||
self.record_table_at(index, table, keys, elapsed, Instant::now());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_at(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration, now: Instant) {
|
||||
self.record_table_at(index, "", keys, elapsed, now);
|
||||
}
|
||||
|
||||
fn record_table_at(
|
||||
&self,
|
||||
index: &str,
|
||||
table: &str,
|
||||
keys: &BTreeMap<String, u8>,
|
||||
elapsed: Duration,
|
||||
now: Instant,
|
||||
) {
|
||||
if keys.is_empty() {
|
||||
return;
|
||||
}
|
||||
let at = self.bucket(now);
|
||||
let mut indexes = self.indexes.lock().unwrap();
|
||||
if !indexes.contains_key(index) && indexes.len() >= MAX_INDEXES {
|
||||
let Some(oldest) = indexes
|
||||
.iter()
|
||||
.min_by_key(|(_, stats)| stats.last)
|
||||
.map(|(name, _)| name.clone())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
indexes.remove(&oldest);
|
||||
}
|
||||
let stats = indexes.entry(index.to_string()).or_default();
|
||||
if !table.is_empty() && stats.table != table {
|
||||
if stats.retired.iter().any(|t| t == table) {
|
||||
return;
|
||||
}
|
||||
if stats.table.is_empty() {
|
||||
stats.table = table.to_string();
|
||||
} else {
|
||||
stats.replace(table);
|
||||
}
|
||||
}
|
||||
stats.last = stats.last.max(at);
|
||||
for (key, kinds) in keys {
|
||||
if !stats.keys.contains_key(key) && stats.keys.len() >= MAX_KEYS_PER_INDEX {
|
||||
// Make room only by dropping a key no query used in the window.
|
||||
let idle = stats
|
||||
.keys
|
||||
.iter()
|
||||
.find(|(_, k)| k.buckets.iter().all(|b| !live(b.at, at)))
|
||||
.map(|(name, _)| name.clone());
|
||||
match idle {
|
||||
Some(idle) => {
|
||||
stats.keys.remove(&idle);
|
||||
}
|
||||
None => {
|
||||
stats.truncated = stats.truncated.max(Some(at));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let entry = stats.keys.entry(key.clone()).or_default();
|
||||
let bucket = &mut entry.buckets[(at % BUCKETS as u64) as usize];
|
||||
if bucket.at != at {
|
||||
*bucket = Bucket {
|
||||
at,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
bucket.queries += 1;
|
||||
bucket.seconds += elapsed.as_secs_f64();
|
||||
bucket.kinds |= kinds;
|
||||
}
|
||||
}
|
||||
|
||||
/// The stats of `index` over the window.
|
||||
pub fn snapshot(&self, index: &str) -> pb::FilterStatsResponse {
|
||||
self.snapshot_at(index, Instant::now())
|
||||
}
|
||||
|
||||
fn snapshot_at(&self, index: &str, now: Instant) -> pb::FilterStatsResponse {
|
||||
let at = self.bucket(now);
|
||||
let window_seconds = now.duration_since(self.start).min(WINDOW).as_secs() as u32;
|
||||
let indexes = self.indexes.lock().unwrap();
|
||||
let Some(stats) = indexes.get(index) else {
|
||||
return pb::FilterStatsResponse {
|
||||
window_seconds,
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
let mut keys: Vec<pb::FilterKeyStats> = stats
|
||||
.keys
|
||||
.iter()
|
||||
.filter_map(|(key, s)| {
|
||||
let (queries, seconds, kinds) = s
|
||||
.buckets
|
||||
.iter()
|
||||
.filter(|b| live(b.at, at))
|
||||
.fold((0, 0.0, 0), |(q, t, k), b| {
|
||||
(q + b.queries, t + b.seconds, k | b.kinds)
|
||||
});
|
||||
(queries > 0).then(|| pb::FilterKeyStats {
|
||||
key: key.clone(),
|
||||
queries,
|
||||
seconds,
|
||||
string_operand: kinds & operand::STRING != 0,
|
||||
number_operand: kinds & operand::NUMBER != 0,
|
||||
bool_operand: kinds & operand::BOOL != 0,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
keys.sort_by(|a, b| b.queries.cmp(&a.queries).then(a.key.cmp(&b.key)));
|
||||
pb::FilterStatsResponse {
|
||||
keys,
|
||||
window_seconds,
|
||||
truncated: stats.truncated.is_some_and(|t| live(t, at)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn keys(pairs: &[(&str, u8)]) -> BTreeMap<String, u8> {
|
||||
pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_queries_and_latency_per_key_within_the_window() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
let ms = Duration::from_millis;
|
||||
stats.record_at("b/i", &keys(&[("author", operand::STRING)]), ms(10), t0);
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[("author", operand::STRING), ("year", operand::NUMBER)]),
|
||||
ms(30),
|
||||
t0 + Duration::from_secs(600),
|
||||
);
|
||||
stats.record_at("b/j", &keys(&[("x", operand::BOOL)]), ms(5), t0);
|
||||
let got = stats.snapshot_at("b/i", t0 + Duration::from_secs(900));
|
||||
assert_eq!(got.window_seconds, 900);
|
||||
assert_eq!(got.keys.len(), 2);
|
||||
assert_eq!(got.keys[0].key, "author");
|
||||
assert_eq!(got.keys[0].queries, 2);
|
||||
assert!((got.keys[0].seconds - 0.04).abs() < 1e-9);
|
||||
assert!(got.keys[0].string_operand && !got.keys[0].number_operand);
|
||||
assert_eq!(got.keys[1].key, "year");
|
||||
assert!(got.keys[1].number_operand);
|
||||
// The first query leaves the window an hour after its bucket.
|
||||
let later = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(1));
|
||||
assert_eq!(later.window_seconds, WINDOW.as_secs() as u32);
|
||||
assert_eq!(later.keys[0].queries, 1);
|
||||
let year = later.keys.iter().find(|k| k.key == "year").unwrap();
|
||||
assert!(year.number_operand);
|
||||
// A string query that left the window no longer marks the key.
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[("author", operand::NUMBER)]),
|
||||
ms(1),
|
||||
t0 + WINDOW + Duration::from_secs(2),
|
||||
);
|
||||
let mixed = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(3));
|
||||
let author = mixed.keys.iter().find(|k| k.key == "author").unwrap();
|
||||
assert!(author.string_operand && author.number_operand);
|
||||
let numeric = stats.snapshot_at("b/i", t0 + WINDOW + Duration::from_secs(700));
|
||||
let author = numeric.keys.iter().find(|k| k.key == "author").unwrap();
|
||||
assert!(!author.string_operand && author.number_operand);
|
||||
let gone = stats.snapshot_at("b/i", t0 + 2 * WINDOW + Duration::from_secs(900));
|
||||
assert!(gone.keys.is_empty());
|
||||
assert!(stats.snapshot_at("other", t0).keys.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indexes_and_keys_are_bounded() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
for i in 0..MAX_KEYS_PER_INDEX + 5 {
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[(&format!("k{i}"), operand::STRING)]),
|
||||
Duration::ZERO,
|
||||
t0,
|
||||
);
|
||||
}
|
||||
let got = stats.snapshot_at("b/i", t0);
|
||||
assert_eq!(got.keys.len(), MAX_KEYS_PER_INDEX);
|
||||
assert!(got.truncated);
|
||||
// A key idle for the whole window makes room for a new one.
|
||||
let later = t0 + WINDOW + BUCKET;
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[("new", operand::STRING)]),
|
||||
Duration::ZERO,
|
||||
later,
|
||||
);
|
||||
let got = stats.snapshot_at("b/i", later);
|
||||
assert!(!got.truncated, "the uncounted keys left the window");
|
||||
assert_eq!(got.keys.len(), 1);
|
||||
assert_eq!(got.keys[0].key, "new");
|
||||
|
||||
for i in 0..MAX_INDEXES + 3 {
|
||||
stats.record_at(
|
||||
&format!("b/{i}"),
|
||||
&keys(&[("k", operand::STRING)]),
|
||||
Duration::ZERO,
|
||||
later,
|
||||
);
|
||||
}
|
||||
assert_eq!(stats.indexes.lock().unwrap().len(), MAX_INDEXES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_record_from_an_earlier_bucket_keeps_the_later_truncation() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
for i in 0..=MAX_KEYS_PER_INDEX {
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[(&format!("k{i}"), operand::STRING)]),
|
||||
Duration::ZERO,
|
||||
t0 + BUCKET,
|
||||
);
|
||||
}
|
||||
// Took its time before the lock while the later bucket recorded.
|
||||
stats.record_at(
|
||||
"b/i",
|
||||
&keys(&[("late", operand::STRING)]),
|
||||
Duration::ZERO,
|
||||
t0,
|
||||
);
|
||||
let got = stats.snapshot_at("b/i", t0 + WINDOW);
|
||||
assert!(
|
||||
got.truncated,
|
||||
"the later bucket's uncounted key is in the window"
|
||||
);
|
||||
assert_eq!(stats.indexes.lock().unwrap()["b/i"].last, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_follow_the_table_behind_an_index_name() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
let author = keys(&[("author", operand::STRING)]);
|
||||
let count = |stats: &FilterStats| {
|
||||
stats
|
||||
.snapshot_at("b/i", t0)
|
||||
.keys
|
||||
.first()
|
||||
.map_or(0, |k| k.queries)
|
||||
};
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 2);
|
||||
// The index was created again: its first query starts over.
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
// A late query of the old table is not counted on the new one.
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
stats.forget("b/i", "");
|
||||
assert_eq!(count(&stats), 0);
|
||||
// A first query records its table: a later table starts over.
|
||||
stats.record_table_at("b/i", "t3", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t4", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_late_query_after_a_drop_is_not_counted_on_the_new_table() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
let author = keys(&[("author", operand::STRING)]);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
stats.forget("b/i", "");
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(stats.snapshot_at("b/i", t0).keys[0].queries, 2);
|
||||
}
|
||||
}
|
||||
@@ -1,817 +0,0 @@
|
||||
//! Indexed metadata keys through the VectorService: PutVectors derives the
|
||||
//! columns, and a query answers the same with or without them.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use arrow_schema::Schema;
|
||||
use lance::Dataset;
|
||||
use lance::dataset::{BatchUDF, NewColumnTransform};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance_index::IndexType;
|
||||
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
|
||||
use tonic::Request;
|
||||
use weed_lance_worker::metadata_index::{self, STATE_FIELD_META, State};
|
||||
use weed_vector_core::query::{self, Query};
|
||||
use weed_vector_core::table::VectorTable;
|
||||
|
||||
use crate::pb::vector::{self as pb, vector_service_server::VectorService};
|
||||
use crate::service::VectorServer;
|
||||
|
||||
const DIM: usize = 8;
|
||||
const ROWS: usize = 3000;
|
||||
const COLUMN: &str = "_mk_0";
|
||||
|
||||
/// How the engine answers a filter once `author` is indexed.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
enum Path {
|
||||
/// Through the column, at most the exact-match limit of rows: exact.
|
||||
Exact,
|
||||
/// Through the column, more rows: the vector index with the column's
|
||||
/// prefilter.
|
||||
Ivf,
|
||||
/// Not through the column.
|
||||
Json,
|
||||
}
|
||||
|
||||
/// S3 Vectors filters, the gateway's translation of each, and its path.
|
||||
const FILTERS: [(&str, &str, Path); 22] = [
|
||||
(
|
||||
"{\"author\":\"alice\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
|
||||
Path::Ivf,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$eq\":\"bob\"}}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"carol\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"carol\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"carol\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"x\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"7\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"7\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"7\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":7}",
|
||||
"(COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) = 7, false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":true}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = 'true', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', 'true'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$in\":[\"alice\",\"bob\",\"x\"]}}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"', '\"x\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"x\"'), false))",
|
||||
Path::Ivf,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$in\":[\"alice\",7]}}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"'), false) OR COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"author\"]') AS DOUBLE) in (7), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '7'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$ne\":\"alice\"}}",
|
||||
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$nin\":[\"alice\",\"bob\"]}}",
|
||||
"NOT (COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"alice\"', '\"bob\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$exists\":false}}",
|
||||
"NOT json_exists(`metadata`, '$[\"author\"]')",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"$and\":[{\"author\":\"alice\"},{\"genre\":\"g1\"}]}",
|
||||
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"alice\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"alice\"'), false)) AND (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false)))",
|
||||
Path::Ivf,
|
||||
),
|
||||
(
|
||||
"{\"$and\":[{\"author\":\"bob\"},{\"year\":{\"$gt\":2005}}]}",
|
||||
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) AND TRY_CAST(json_extract(`metadata`, '$[\"year\"]') AS DOUBLE) > 2005)",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"$or\":[{\"author\":\"bob\"},{\"genre\":\"g2\"}]}",
|
||||
"((COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"bob\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"bob\"'), false)) OR (COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g2\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g2\"'), false)))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"o'neil \\\"q\\\" \\\\ \\n\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"o''neil \\\"q\\\" \\\\ \\n\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"o''neil \\\"q\\\" \\\\ \\n\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"nobody\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"nobody\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"nobody\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
("{\"author\":{\"$in\":[]}}", "false", Path::Json),
|
||||
(
|
||||
"{\"genre\":\"g1\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"genre\"]') = '\"g1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"genre\"]', '\"g1\"'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"u5\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"u5\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
(
|
||||
"{\"author\":\"ctl\\u0001x\"}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') = '\"ctl\\u{1}x\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"ctl\\u0001x\"'), false))",
|
||||
Path::Json,
|
||||
),
|
||||
(
|
||||
"{\"author\":{\"$in\":[\"u5\",\"u6\"]}}",
|
||||
"(COALESCE(json_extract(`metadata`, '$[\"author\"]') in ('\"u5\"', '\"u6\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u5\"'), false) OR COALESCE(json_array_contains(`metadata`, '$[\"author\"]', '\"u6\"'), false))",
|
||||
Path::Exact,
|
||||
),
|
||||
];
|
||||
|
||||
fn rng(seed: &mut u64) -> f32 {
|
||||
*seed = seed
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
((*seed >> 33) as f32 / (1u64 << 31) as f32) - 0.5
|
||||
}
|
||||
|
||||
fn vector_of(i: usize) -> Vec<f32> {
|
||||
let mut centre = (i % 16) as u64 + 1;
|
||||
let mut noise = i as u64 * 7919 + 13;
|
||||
(0..DIM)
|
||||
.map(|_| rng(&mut centre) * 4.0 + rng(&mut noise))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn author(i: usize) -> Option<serde_json::Value> {
|
||||
use serde_json::json;
|
||||
Some(match i % 100 {
|
||||
0..=9 => json!("alice"),
|
||||
10 => json!("bob"),
|
||||
11 => json!(["carol", "x"]),
|
||||
12 => json!(7),
|
||||
13 => json!("7"),
|
||||
14 => serde_json::Value::Null,
|
||||
15 => return None,
|
||||
16 => json!(true),
|
||||
17 => json!("o'neil \"q\" \\ \n"),
|
||||
18 => json!("ctl\u{1}x"),
|
||||
19 => json!(["alice", 3]),
|
||||
_ => json!(format!("u{}", i % 37)),
|
||||
})
|
||||
}
|
||||
|
||||
fn row(i: usize, author: Option<serde_json::Value>) -> pb::Vector {
|
||||
let mut meta = serde_json::json!({"genre": format!("g{}", i % 3), "year": 2000 + i % 10});
|
||||
if let Some(author) = author {
|
||||
meta["author"] = author;
|
||||
}
|
||||
let serde_json::Value::Object(meta) = meta else {
|
||||
unreachable!()
|
||||
};
|
||||
pb::Vector {
|
||||
key: format!("k{i}"),
|
||||
data: vector_of(i),
|
||||
metadata: Some(crate::service::json_to_struct(meta)),
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
server: VectorServer,
|
||||
exact_off: VectorServer,
|
||||
table: pb::Table,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn new() -> Self {
|
||||
let bucket = format!(
|
||||
"mk-{:x}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
);
|
||||
let table = pb::Table {
|
||||
id: vec![bucket.clone(), "vector".into(), "x".into()],
|
||||
location: format!("shared-memory://{bucket}/vector/x"),
|
||||
..Default::default()
|
||||
};
|
||||
let fixture = Self {
|
||||
server: VectorServer::new().with_schemes(&["shared-memory"]),
|
||||
exact_off: VectorServer::new()
|
||||
.with_schemes(&["shared-memory"])
|
||||
.with_exact_match_limit(Some(0)),
|
||||
table,
|
||||
};
|
||||
fixture
|
||||
.server
|
||||
.create_index(Request::new(pb::CreateIndexRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
dimension: DIM as u32,
|
||||
distance_metric: pb::DistanceMetric::L2 as i32,
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn put(&self, vectors: Vec<pb::Vector>) {
|
||||
self.server
|
||||
.put_vectors(Request::new(pb::PutVectorsRequest {
|
||||
table: Some(self.table.clone()),
|
||||
vectors,
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn dataset(&self, version: u64) -> Dataset {
|
||||
let dataset = Dataset::open(&self.table.location).await.unwrap();
|
||||
if version == 0 {
|
||||
dataset
|
||||
} else {
|
||||
dataset.checkout_version(version).await.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds `author`'s column, indexes it and marks it ready, returning the
|
||||
/// version before it was ready.
|
||||
async fn index_author(&self) -> u64 {
|
||||
let mut dataset = self.dataset(0).await;
|
||||
let output = Arc::new(Schema::new(vec![metadata_index::field(
|
||||
COLUMN,
|
||||
"author",
|
||||
State::Building,
|
||||
)]));
|
||||
let schema = output.clone();
|
||||
dataset
|
||||
.add_columns(
|
||||
NewColumnTransform::BatchUDF(BatchUDF {
|
||||
mapper: Box::new(move |batch: &RecordBatch| {
|
||||
let column = metadata_index::column_from_metadata(
|
||||
batch.column_by_name("metadata").unwrap().as_ref(),
|
||||
"author",
|
||||
)
|
||||
.map_err(|e| lance::Error::invalid_input(e.to_string()))?;
|
||||
Ok(RecordBatch::try_new(schema.clone(), vec![column])?)
|
||||
}),
|
||||
output_schema: output,
|
||||
result_checkpoint: None,
|
||||
}),
|
||||
Some(vec!["metadata".to_string()]),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&[COLUMN],
|
||||
IndexType::LabelList,
|
||||
Some(format!("{COLUMN}_idx")),
|
||||
&ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let building = dataset.version().version;
|
||||
self.set_state(State::Ready).await;
|
||||
building
|
||||
}
|
||||
|
||||
async fn set_state(&self, state: State) -> u64 {
|
||||
let mut dataset = self.dataset(0).await;
|
||||
dataset
|
||||
.update_field_metadata()
|
||||
.update(COLUMN, [(STATE_FIELD_META, state.as_str())])
|
||||
.unwrap()
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.version().version
|
||||
}
|
||||
|
||||
async fn query(
|
||||
&self,
|
||||
server: &VectorServer,
|
||||
version: u64,
|
||||
filter: &str,
|
||||
k: u32,
|
||||
v: &[f32],
|
||||
) -> Vec<(String, f32)> {
|
||||
let mut table = self.table.clone();
|
||||
table.version = version;
|
||||
server
|
||||
.query_vectors(Request::new(pb::QueryVectorsRequest {
|
||||
table: Some(table),
|
||||
query_vector: v.to_vec(),
|
||||
top_k: k,
|
||||
filter: filter.to_string(),
|
||||
return_distance: true,
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner()
|
||||
.vectors
|
||||
.into_iter()
|
||||
.map(|r| (r.key, r.distance))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Exact filtered top-k at `version`.
|
||||
async fn flat(&self, version: u64, filter: &str, k: u32, v: &[f32]) -> Vec<(String, f32)> {
|
||||
let dataset = self.dataset(version).await;
|
||||
let table = VectorTable::resolve(&dataset, None, Some("key")).unwrap();
|
||||
query::flat(
|
||||
&dataset,
|
||||
&table,
|
||||
&Query {
|
||||
vector: v.to_vec(),
|
||||
k: k as usize,
|
||||
filter: Some(filter.to_string()),
|
||||
postfilter: false,
|
||||
distance: lance_linalg::distance::DistanceType::L2,
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|h| (h.key, h.distance))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every filter answers through the column as through the JSON at
|
||||
/// `json` (the same rows and indexes, the column not ready), and an
|
||||
/// exact answer as the exact filtered top-k.
|
||||
async fn compare(&self, json: u64, ready: u64) {
|
||||
for q in [7, 1234] {
|
||||
let v = vector_of(q);
|
||||
for (s3, filter, path) in FILTERS {
|
||||
for k in [10, 400] {
|
||||
let what = format!("{s3} k={k} q={q}");
|
||||
let exact = self.flat(json, filter, k, &v).await;
|
||||
assert_same(&self.flat(ready, filter, k, &v).await, &exact, &what);
|
||||
let json_ann = self.query(&self.server, json, filter, k, &v).await;
|
||||
let off = self.query(&self.exact_off, ready, filter, k, &v).await;
|
||||
assert_same(&off, &json_ann, &format!("{what}: indexed prefilter"));
|
||||
let got = self.query(&self.server, ready, filter, k, &v).await;
|
||||
let want = if path == Path::Exact {
|
||||
&exact
|
||||
} else {
|
||||
&json_ann
|
||||
};
|
||||
assert_same(&got, want, &format!("{what}: {path:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_same(got: &[(String, f32)], want: &[(String, f32)], what: &str) {
|
||||
assert_eq!(got.len(), want.len(), "{what}: {got:?} vs {want:?}");
|
||||
// Ties may come back in either order: the same keys, and the same
|
||||
// distance at each rank.
|
||||
fn keys(hits: &[(String, f32)]) -> Vec<&str> {
|
||||
let mut keys: Vec<&str> = hits.iter().map(|(k, _)| k.as_str()).collect();
|
||||
keys.sort();
|
||||
keys
|
||||
}
|
||||
assert_eq!(keys(got), keys(want), "{what}: {got:?} vs {want:?}");
|
||||
for (g, w) in got.iter().zip(want) {
|
||||
let same = (g.1 - w.1).abs() < 1e-4 || (g.1.is_infinite() && w.1.is_infinite());
|
||||
assert!(same, "{what}: {got:?} vs {want:?}");
|
||||
if g.0 != w.0 {
|
||||
let tie = want
|
||||
.iter()
|
||||
.any(|(k, d)| k == &g.0 && (d - w.1).abs() < 1e-4);
|
||||
assert!(
|
||||
tie,
|
||||
"{what}: {} at {} is no tie: {got:?} vs {want:?}",
|
||||
g.0, g.1
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed and JSON answers agree on mixed types, missing keys, arrays,
|
||||
/// null, untouched operators, and after updates, deletes and rows no
|
||||
/// scalar or vector index covers yet.
|
||||
#[tokio::test]
|
||||
async fn indexed_answers_match_the_json_path() {
|
||||
let fixture = Fixture::new().await;
|
||||
for start in (0..ROWS).step_by(300) {
|
||||
fixture
|
||||
.put((start..start + 300).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
}
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let json = fixture.index_author().await;
|
||||
let schema: Schema = fixture.dataset(0).await.schema().into();
|
||||
assert_eq!(
|
||||
metadata_index::indexed_keys(&schema)[0].state,
|
||||
Some(State::Ready)
|
||||
);
|
||||
fixture.compare(json, 0).await;
|
||||
|
||||
// Updates move rows between values, deletes drop some, and new rows land
|
||||
// in fragments neither index covers; PutVectors fills the column.
|
||||
fixture
|
||||
.put(
|
||||
(0..60)
|
||||
.map(|i| row(i, Some(serde_json::json!(["moved", "bob"]))))
|
||||
.chain((ROWS..ROWS + 200).map(|i| row(i, author(i))))
|
||||
.collect(),
|
||||
)
|
||||
.await;
|
||||
fixture
|
||||
.server
|
||||
.delete_vectors(Request::new(pb::DeleteVectorsRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
keys: (100..130).map(|i| format!("k{i}")).collect(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let json = fixture.set_state(State::Building).await;
|
||||
let ready = fixture.set_state(State::Ready).await;
|
||||
fixture.compare(json, ready).await;
|
||||
let moved = r#"(COALESCE(json_extract(`metadata`, '$["author"]') = '"moved"', false) OR COALESCE(json_array_contains(`metadata`, '$["author"]', '"moved"'), false))"#;
|
||||
let hits = fixture
|
||||
.query(&fixture.server, ready, moved, 100, &vector_of(0))
|
||||
.await;
|
||||
assert_eq!(hits.len(), 60);
|
||||
}
|
||||
|
||||
/// The planner counts and scores per scope: an exact answer over a fragment
|
||||
/// subset is that subset's exact top-k.
|
||||
#[tokio::test]
|
||||
async fn a_scoped_query_is_exact_within_its_fragments() {
|
||||
let fixture = Fixture::new().await;
|
||||
for start in (0..ROWS).step_by(300) {
|
||||
fixture
|
||||
.put((start..start + 300).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
}
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 16, false)
|
||||
.await
|
||||
.unwrap();
|
||||
let json = fixture.index_author().await;
|
||||
let ready = fixture.dataset(0).await;
|
||||
let table = VectorTable::resolve(&ready, None, Some("key")).unwrap();
|
||||
let ids: Vec<u64> = ready.fragments().iter().map(|f| f.id).take(4).collect();
|
||||
let filter = FILTERS[1].1;
|
||||
let scoped = || Query {
|
||||
vector: vector_of(5),
|
||||
k: 10,
|
||||
filter: Some(filter.to_string()),
|
||||
postfilter: false,
|
||||
distance: lance_linalg::distance::DistanceType::L2,
|
||||
nprobes: None,
|
||||
index_segments: None,
|
||||
fragments: Some(ids.clone()),
|
||||
exact_match_limit: None,
|
||||
};
|
||||
let got = query::ann(&ready, &table, &scoped()).await.unwrap();
|
||||
let old = fixture.dataset(json).await;
|
||||
let want = query::flat(&old, &table, &scoped()).await.unwrap();
|
||||
assert_eq!(want.len(), 10);
|
||||
let pairs = |hits: Vec<weed_vector_core::Hit>| {
|
||||
hits.into_iter()
|
||||
.map(|h| (h.key, h.distance))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
assert_same(&pairs(got), &pairs(want), "scoped bob");
|
||||
}
|
||||
|
||||
/// The engine counts each index's filtered queries by key.
|
||||
#[tokio::test]
|
||||
async fn queries_are_counted_by_filter_key() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..50).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
for (_, filter, _) in &FILTERS[..3] {
|
||||
fixture
|
||||
.query(&fixture.server, 0, filter, 5, &vector_of(1))
|
||||
.await;
|
||||
}
|
||||
let year = FILTERS[13].1;
|
||||
fixture
|
||||
.query(&fixture.server, 0, year, 5, &vector_of(1))
|
||||
.await;
|
||||
let stats = fixture
|
||||
.server
|
||||
.filter_stats(Request::new(pb::FilterStatsRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let author = stats.keys.iter().find(|k| k.key == "author").unwrap();
|
||||
assert_eq!(author.queries, 4);
|
||||
assert!(author.string_operand && !author.number_operand);
|
||||
let year = stats.keys.iter().find(|k| k.key == "year").unwrap();
|
||||
assert_eq!(year.queries, 1);
|
||||
assert!(year.number_operand && !year.string_operand);
|
||||
}
|
||||
|
||||
/// A PutVectors that opened the table before its key column was added and
|
||||
/// commits after: Lance rebases the write and its rows hold NULL in the new
|
||||
/// column, which the write then fills from the rows' metadata.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_write_that_raced_the_column_add_fills_the_column() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..300).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let unhealed = fixture.dataset(0).await;
|
||||
let healed = fixture.dataset(0).await;
|
||||
fixture.index_author().await;
|
||||
let nulls = |dataset: Dataset| async move {
|
||||
dataset
|
||||
.count_rows(Some(format!("{COLUMN} IS NULL")))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
let rows: Vec<pb::Vector> = (250..400).map(|i| row(i, author(i))).collect();
|
||||
|
||||
let written = crate::service::merge_vectors(unhealed, &rows)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(written.schema().field(COLUMN).is_some());
|
||||
assert!(nulls(written).await >= 100, "the race is reproduced");
|
||||
|
||||
let written = crate::service::write_vectors(healed, &rows).await.unwrap();
|
||||
assert_eq!(nulls(written.clone()).await, 0);
|
||||
let filter = FILTERS[1].1;
|
||||
let latest = written.version().version;
|
||||
let v = vector_of(5);
|
||||
assert_same(
|
||||
&fixture
|
||||
.query(&fixture.server, latest, filter, 400, &v)
|
||||
.await,
|
||||
&fixture.flat(latest, filter, 400, &v).await,
|
||||
"bob after the race",
|
||||
);
|
||||
assert_eq!(fixture.flat(latest, filter, 400, &v).await.len(), 4);
|
||||
}
|
||||
|
||||
/// A write names no version: one at a version from before a key's column
|
||||
/// add would commit rows without the column.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_write_at_an_older_version_is_refused() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..10).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let before = fixture.dataset(0).await.version().version;
|
||||
fixture.index_author().await;
|
||||
let mut table = fixture.table.clone();
|
||||
table.version = before;
|
||||
let refused = fixture
|
||||
.server
|
||||
.put_vectors(Request::new(pb::PutVectorsRequest {
|
||||
table: Some(table),
|
||||
vectors: (10..20).map(|i| row(i, author(i))).collect(),
|
||||
}))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(refused.code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
/// Two writes updating the same keys raced the column add, and the first
|
||||
/// one's fill commits last: each write's rows get the values of its own
|
||||
/// metadata, not the ones derived before it or from the other write.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_write_that_raced_the_column_add_derives_its_updated_rows() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..300).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let first = fixture.dataset(0).await;
|
||||
let second = fixture.dataset(0).await;
|
||||
let written: Schema = first.schema().into();
|
||||
fixture.index_author().await;
|
||||
let rows = |name: &str| -> Vec<pb::Vector> {
|
||||
(250..300)
|
||||
.map(|i| row(i, Some(serde_json::json!(name))))
|
||||
.collect()
|
||||
};
|
||||
let keys: Vec<String> = (250..300).map(|i| format!("k{i}")).collect();
|
||||
|
||||
let first = crate::service::merge_vectors(first, &rows("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
let second = crate::service::merge_vectors(second, &rows("bob"))
|
||||
.await
|
||||
.unwrap();
|
||||
crate::service::heal_columns(second, &written, &keys)
|
||||
.await
|
||||
.unwrap();
|
||||
crate::service::heal_columns(first, &written, &keys)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let latest = fixture.dataset(0).await;
|
||||
let count = |filter: String| {
|
||||
let latest = latest.clone();
|
||||
async move { latest.count_rows(Some(filter)).await.unwrap() }
|
||||
};
|
||||
assert_eq!(count(format!("{COLUMN} IS NULL")).await, 0);
|
||||
for (name, filter) in [("alice", FILTERS[0].1), ("bob", FILTERS[1].1)] {
|
||||
assert_eq!(
|
||||
count(format!("array_has({COLUMN}, '{name}')")).await,
|
||||
count(filter.to_string()).await,
|
||||
"{name}"
|
||||
);
|
||||
}
|
||||
let mut got = fixture
|
||||
.server
|
||||
.get_vectors(Request::new(pb::GetVectorsRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
keys: keys.clone(),
|
||||
return_data: true,
|
||||
return_metadata: true,
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner()
|
||||
.vectors;
|
||||
got.sort_by_key(|v| v.key[1..].parse::<usize>().unwrap());
|
||||
assert_eq!(got, rows("bob"));
|
||||
}
|
||||
|
||||
/// Operators ask through the engine; the table records the request and,
|
||||
/// once the worker has done its part, the key's state, coverage and size.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn keys_are_requested_and_listed_through_the_engine() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..600).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let update = |key: &str, remove: bool| {
|
||||
fixture
|
||||
.server
|
||||
.update_metadata_key(Request::new(pb::UpdateMetadataKeyRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
key: key.to_string(),
|
||||
remove,
|
||||
}))
|
||||
};
|
||||
let listed = update("author", false).await.unwrap().into_inner();
|
||||
assert_eq!(listed.keys.len(), 1);
|
||||
assert_eq!(listed.keys[0].state, "requested");
|
||||
assert_eq!(listed.keys[0].rows, 600);
|
||||
// A retry of a committed request answers as the first one did.
|
||||
let again = update("author", false).await.unwrap().into_inner();
|
||||
assert_eq!(again.keys.len(), 1);
|
||||
assert_eq!(again.keys[0].state, "requested");
|
||||
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
let key = metadata_index::add_column(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::finish_after(&mut dataset, &key, 1_000_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
let listed = fixture
|
||||
.server
|
||||
.metadata_keys(Request::new(pb::MetadataKeysRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
assert_eq!((listed.rows, listed.vector_index.as_str()), (600, ""));
|
||||
let author = &listed.keys[0];
|
||||
assert_eq!(
|
||||
(
|
||||
author.key.as_str(),
|
||||
author.state.as_str(),
|
||||
author.column.as_str()
|
||||
),
|
||||
("author", "ready", "_mk_0")
|
||||
);
|
||||
assert_eq!(
|
||||
(author.indexed_rows, author.rows, author.segments),
|
||||
(600, 600, 1)
|
||||
);
|
||||
assert!(author.index_bytes > 0);
|
||||
|
||||
// Off at once: the next query reads the JSON again.
|
||||
let listed = update("author", true).await.unwrap().into_inner();
|
||||
assert_eq!(listed.keys[0].state, "off");
|
||||
let dataset = fixture.dataset(0).await;
|
||||
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
|
||||
// Removing a key that is not indexed needs nothing.
|
||||
let nobody = update("nobody", true).await.unwrap().into_inner();
|
||||
assert_eq!(nobody.keys.len(), 1);
|
||||
assert_eq!(
|
||||
update("author", false).await.unwrap_err().code(),
|
||||
tonic::Code::FailedPrecondition,
|
||||
"being removed"
|
||||
);
|
||||
}
|
||||
|
||||
/// DeleteIndex purges the table's directory and an index created again
|
||||
/// under the same name reuses it: the same running engine serves nothing of
|
||||
/// the old table, neither rows, nor its indexed key, nor its filter stats.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_dropped_and_recreated_index_serves_nothing_old() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..600).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 4, false)
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.index_author().await;
|
||||
let bob = FILTERS[1].1;
|
||||
let old = fixture
|
||||
.query(&fixture.server, 0, bob, 50, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(old.len(), 6);
|
||||
let server = &fixture.server;
|
||||
let stats = || {
|
||||
server.filter_stats(Request::new(pb::FilterStatsRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
};
|
||||
assert_eq!(stats().await.unwrap().into_inner().keys[0].queries, 1);
|
||||
|
||||
// What the catalog's DeleteTable does, then CreateIndex again.
|
||||
let (store, path) = lance::io::ObjectStore::from_uri(&fixture.table.location)
|
||||
.await
|
||||
.unwrap();
|
||||
store.remove_dir_all(path).await.unwrap();
|
||||
fixture
|
||||
.server
|
||||
.create_index(Request::new(pb::CreateIndexRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
dimension: DIM as u32,
|
||||
distance_metric: pb::DistanceMetric::L2 as i32,
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = fixture
|
||||
.server
|
||||
.metadata_keys(Request::new(pb::MetadataKeysRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
assert!(keys.keys.is_empty());
|
||||
assert!(
|
||||
stats().await.unwrap().into_inner().keys.is_empty(),
|
||||
"the old table's filter stats"
|
||||
);
|
||||
|
||||
fixture
|
||||
.put((1000..1100).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let new = |hits: &[(String, f32)]| {
|
||||
hits.iter().all(|(k, _)| {
|
||||
k[1..]
|
||||
.parse::<usize>()
|
||||
.is_ok_and(|i| (1000..1100).contains(&i))
|
||||
})
|
||||
};
|
||||
let unfiltered = fixture
|
||||
.query(&fixture.server, 0, "", 200, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(unfiltered.len(), 100);
|
||||
assert!(new(&unfiltered), "{unfiltered:?}");
|
||||
let bob_now = fixture
|
||||
.query(&fixture.server, 0, bob, 50, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(bob_now.len(), 1);
|
||||
assert!(new(&bob_now), "{bob_now:?}");
|
||||
let author = stats().await.unwrap().into_inner().keys;
|
||||
assert_eq!((author.len(), author[0].queries), (1, 1));
|
||||
}
|
||||
@@ -1,12 +1,5 @@
|
||||
//! The VectorService gRPC server the S3 gateway talks to.
|
||||
|
||||
// PutVectors awaits two Lance merge_insert futures; proving the handler
|
||||
// future Send takes deeper trait recursion than the default.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
pub mod filter_stats;
|
||||
#[cfg(test)]
|
||||
mod indexed_keys_tests;
|
||||
pub mod metrics;
|
||||
pub mod observed;
|
||||
pub mod read_cache;
|
||||
|
||||
@@ -52,11 +52,6 @@ struct Args {
|
||||
/// tenth of the index's partitions, at least 8; more under a filter.
|
||||
#[arg(long, env = "WEED_VECTOR_NPROBES")]
|
||||
nprobes: Option<std::num::NonZeroUsize>,
|
||||
/// A filtered query whose indexed metadata keys match at most this many
|
||||
/// rows scores them exactly instead of searching the vector index; 0
|
||||
/// never does.
|
||||
#[arg(long, env = "WEED_VECTOR_EXACT_MATCH_LIMIT", default_value_t = weed_vector_core::query::DEFAULT_EXACT_MATCH_LIMIT)]
|
||||
exact_match_limit: usize,
|
||||
/// The cluster's security.toml: [grpc.vector] for this server,
|
||||
/// [grpc.client] for the masters and filers, https.client for volume
|
||||
/// servers under HTTPS, jwt.signing.read.key for signed volume reads,
|
||||
@@ -175,7 +170,6 @@ async fn main() -> Result<()> {
|
||||
.then(|| ReadCache::new(args.read_cache_mb << 20).with_metrics(metrics.clone()));
|
||||
let mut service = VectorServer::new()
|
||||
.with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get))
|
||||
.with_exact_match_limit(Some(args.exact_match_limit))
|
||||
.with_lance_caches(args.index_cache_mb << 20, args.metadata_cache_mb << 20)
|
||||
.with_storage(reads);
|
||||
if let Some(cache) = &cache {
|
||||
|
||||
@@ -10,10 +10,6 @@ use tonic_health::{ServingStatus, server::HealthReporter};
|
||||
|
||||
use crate::read_cache::ReadCache;
|
||||
|
||||
/// The gRPC health service an engine reports serving when its writes derive
|
||||
/// the indexed metadata key columns.
|
||||
pub const METADATA_COLUMNS_CAPABILITY: &str = "weed.vector.capability.metadata_columns";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Metrics {
|
||||
pub registry: Registry,
|
||||
@@ -124,11 +120,6 @@ impl Metrics {
|
||||
health
|
||||
.set_service_status("vector_pb.VectorService", status)
|
||||
.await;
|
||||
// A capability, not readiness: the admin refuses to index a
|
||||
// metadata key while an engine without it could write the table.
|
||||
health
|
||||
.set_service_status(METADATA_COLUMNS_CAPABILITY, ServingStatus::Serving)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,33 +78,6 @@ impl VectorService for ObservedService {
|
||||
}
|
||||
result
|
||||
}
|
||||
async fn metadata_keys(
|
||||
&self,
|
||||
request: Request<pb::MetadataKeysRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let mut in_flight = self.metrics.start("MetadataKeys");
|
||||
let result = self.inner.metadata_keys(request).await;
|
||||
in_flight.complete(result.as_ref().err().map(|e| e.code()));
|
||||
result
|
||||
}
|
||||
async fn update_metadata_key(
|
||||
&self,
|
||||
request: Request<pb::UpdateMetadataKeyRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let mut in_flight = self.metrics.start("UpdateMetadataKey");
|
||||
let result = self.inner.update_metadata_key(request).await;
|
||||
in_flight.complete(result.as_ref().err().map(|e| e.code()));
|
||||
result
|
||||
}
|
||||
async fn filter_stats(
|
||||
&self,
|
||||
request: Request<pb::FilterStatsRequest>,
|
||||
) -> Result<Response<pb::FilterStatsResponse>, Status> {
|
||||
let mut in_flight = self.metrics.start("FilterStats");
|
||||
let result = self.inner.filter_stats(request).await;
|
||||
in_flight.complete(result.as_ref().err().map(|e| e.code()));
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,9 +19,7 @@ use weed_lance_worker::catalog::TableDescription;
|
||||
use weed_lance_worker::dataset;
|
||||
use weed_lance_worker::direct_read::{DirectReads, Wrappers};
|
||||
use weed_lance_worker::jobs::vector_index::is_vector_table;
|
||||
use weed_lance_worker::metadata_index;
|
||||
use weed_vector_core::index;
|
||||
use weed_vector_core::indexed_filter;
|
||||
use weed_vector_core::maintenance::SharedLeases;
|
||||
use weed_vector_core::query::{self, Query};
|
||||
use weed_vector_core::table::VectorTable;
|
||||
@@ -169,13 +167,14 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
|
||||
};
|
||||
let vectors_array =
|
||||
FixedSizeListArray::try_new(item_field.clone(), dimension as i32, values, None)?;
|
||||
let json: Vec<Option<serde_json::Value>> = vectors
|
||||
let metas = vectors
|
||||
.iter()
|
||||
.map(|v| v.metadata.as_ref().map(struct_to_json))
|
||||
.collect();
|
||||
let metas = json
|
||||
.iter()
|
||||
.map(|m| m.as_ref().map(serde_json::to_string).transpose())
|
||||
.map(|v| {
|
||||
v.metadata
|
||||
.as_ref()
|
||||
.map(|m| serde_json::to_string(&struct_to_json(m)))
|
||||
.transpose()
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
// The stored column is lance's jsonb (LargeBinary) once the field carries
|
||||
// the arrow.json extension; encode the strings, else keep the Utf8 batch.
|
||||
@@ -188,188 +187,11 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Re
|
||||
}
|
||||
_ => Arc::new(StringArray::from(metas)),
|
||||
};
|
||||
let (keys, vectors_array): (ArrayRef, ArrayRef) = (Arc::new(keys), Arc::new(vectors_array));
|
||||
// An indexed metadata key's column is derived from the row's metadata,
|
||||
// whatever the key's state.
|
||||
let columns = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| match field.name().as_str() {
|
||||
KEY_COLUMN => Ok(keys.clone()),
|
||||
VECTOR_COLUMN => Ok(vectors_array.clone()),
|
||||
METADATA_COLUMN => Ok(metadata_array.clone()),
|
||||
name => match metadata_index::indexed_key(field) {
|
||||
Some(indexed) => Ok(metadata_index::column_from_json(
|
||||
json.iter().map(Option::as_ref),
|
||||
&indexed.key,
|
||||
)),
|
||||
None => bail!("the table has a column {name} that PutVectors cannot fill"),
|
||||
},
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
RecordBatch::try_new(schema, columns).map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Upserts `vectors` by key into `dataset`, the version the request opened,
|
||||
/// then fills the indexed-key columns a concurrent column add left empty.
|
||||
pub(crate) async fn write_vectors(
|
||||
dataset: Dataset,
|
||||
vectors: &[pb::Vector],
|
||||
) -> Result<Dataset, Status> {
|
||||
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
|
||||
let dataset = merge_vectors(dataset, vectors).await?;
|
||||
let keys: Vec<String> = vectors.iter().map(|v| v.key.clone()).collect();
|
||||
heal_columns(dataset, &written, &keys)
|
||||
.await
|
||||
.map_err(internal)
|
||||
}
|
||||
|
||||
/// The upsert itself, in the schema of the version `dataset` is at.
|
||||
pub(crate) async fn merge_vectors(
|
||||
dataset: Dataset,
|
||||
vectors: &[pb::Vector],
|
||||
) -> Result<Dataset, Status> {
|
||||
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
|
||||
.map_err(invalid)?
|
||||
.dimension;
|
||||
let batch = batch_of(
|
||||
vectors,
|
||||
dimension,
|
||||
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
|
||||
RecordBatch::try_new(
|
||||
schema,
|
||||
vec![Arc::new(keys), Arc::new(vectors_array), metadata_array],
|
||||
)
|
||||
.map_err(invalid)?;
|
||||
// Upsert by key in one merge commit: no window where the old rows are
|
||||
// already deleted, and a losing writer fails the request instead of
|
||||
// leaving duplicate keys.
|
||||
let mut builder = MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
|
||||
.map_err(internal)?;
|
||||
builder.when_matched(WhenMatched::UpdateAll);
|
||||
builder.when_not_matched(WhenNotMatched::InsertAll);
|
||||
let (dataset, _stats) = builder
|
||||
.try_build()
|
||||
.map_err(internal)?
|
||||
.execute_batches(vec![batch])
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
Ok(Arc::unwrap_or_clone(dataset))
|
||||
}
|
||||
|
||||
/// Attempts [`heal_columns`] makes against writes committing meanwhile.
|
||||
const HEAL_ATTEMPTS: usize = 10;
|
||||
|
||||
/// Lance rebases a write onto a column add committed after the write opened
|
||||
/// its version: the rows it inserted hold NULL in the new column and the
|
||||
/// rows it updated keep the value derived from their old metadata. Derive
|
||||
/// such columns again for the rows of `keys` from their stored metadata.
|
||||
pub(crate) async fn heal_columns(
|
||||
mut dataset: Dataset,
|
||||
written: &Schema,
|
||||
keys: &[String],
|
||||
) -> Result<Dataset> {
|
||||
for _ in 0..HEAL_ATTEMPTS {
|
||||
let schema: Schema = dataset.schema().into();
|
||||
let missing: Vec<metadata_index::IndexedKey> = metadata_index::indexed_keys(&schema)
|
||||
.into_iter()
|
||||
.filter(|k| written.field_with_name(&k.column).is_err())
|
||||
.collect();
|
||||
if missing.is_empty() || keys.is_empty() {
|
||||
return Ok(dataset);
|
||||
}
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.filter(&key_list(keys))?
|
||||
.project(&[KEY_COLUMN, METADATA_COLUMN])?;
|
||||
let rows: Vec<RecordBatch> = scanner.try_into_stream().await?.try_collect().await?;
|
||||
let mut fields = vec![schema.field_with_name(KEY_COLUMN)?.clone()];
|
||||
for k in &missing {
|
||||
fields.push(schema.field_with_name(&k.column)?.clone());
|
||||
}
|
||||
let patch_schema = Arc::new(Schema::new(fields));
|
||||
let mut patches = Vec::new();
|
||||
for batch in rows.iter().filter(|b| b.num_rows() > 0) {
|
||||
let (Some(keys), Some(metadata)) = (
|
||||
batch.column_by_name(KEY_COLUMN),
|
||||
batch.column_by_name(METADATA_COLUMN),
|
||||
) else {
|
||||
bail!("the scan did not return keys and metadata");
|
||||
};
|
||||
let mut columns = vec![keys.clone()];
|
||||
for k in &missing {
|
||||
columns.push(metadata_index::column_from_metadata(
|
||||
metadata.as_ref(),
|
||||
&k.key,
|
||||
)?);
|
||||
}
|
||||
patches.push(RecordBatch::try_new(patch_schema.clone(), columns)?);
|
||||
}
|
||||
if patches.is_empty() {
|
||||
return Ok(dataset);
|
||||
}
|
||||
// Lance's own retry would join these values onto rows a later write
|
||||
// replaced; derive them again from the version that write left.
|
||||
let mut builder =
|
||||
MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec![KEY_COLUMN.to_string()])?;
|
||||
builder
|
||||
.when_matched(WhenMatched::UpdateAll)
|
||||
.when_not_matched(WhenNotMatched::DoNothing)
|
||||
.conflict_retries(0);
|
||||
match builder.try_build()?.execute_batches(patches).await {
|
||||
Ok((healed, _stats)) => {
|
||||
tracing::info!(
|
||||
rows = rows.iter().map(|b| b.num_rows()).sum::<usize>(),
|
||||
"derived indexed metadata key columns a concurrent column add left stale"
|
||||
);
|
||||
return Ok(Arc::unwrap_or_clone(healed));
|
||||
}
|
||||
Err(lance::Error::TooMuchWriteContention { .. }) => dataset.checkout_latest().await?,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
bail!("writes kept replacing the rows while their indexed key columns were derived")
|
||||
}
|
||||
|
||||
/// The indexed metadata keys `dataset` records, requested ones first.
|
||||
async fn metadata_keys(dataset: &Dataset) -> Result<pb::MetadataKeysResponse> {
|
||||
let rows: u64 = dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.map(|f| f.num_rows().unwrap_or(0) as u64)
|
||||
.sum();
|
||||
let mut keys: Vec<pb::MetadataKey> = metadata_index::requests(dataset)
|
||||
.into_iter()
|
||||
.map(|key| pb::MetadataKey {
|
||||
key,
|
||||
state: "requested".to_string(),
|
||||
rows,
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
for key in metadata_index::keys_of(dataset) {
|
||||
let segments = metadata_index::segments(dataset, &key.column).await?;
|
||||
keys.push(pb::MetadataKey {
|
||||
state: key.state.map_or("unknown", |s| s.as_str()).to_string(),
|
||||
indexed_rows: segments.iter().map(|s| s.rows).sum(),
|
||||
index_bytes: segments.iter().map(|s| s.bytes).sum(),
|
||||
segments: segments.len() as u32,
|
||||
rows,
|
||||
key: key.key,
|
||||
column: key.column,
|
||||
});
|
||||
}
|
||||
let vector_index = match index::index_names(dataset, VECTOR_COLUMN).await?.first() {
|
||||
Some(name) => {
|
||||
weed_lance_worker::vector_index::index_settings(dataset, name)
|
||||
.await?
|
||||
.index_type
|
||||
}
|
||||
None => String::new(),
|
||||
};
|
||||
Ok(pb::MetadataKeysResponse {
|
||||
keys,
|
||||
version: dataset.version().version,
|
||||
rows,
|
||||
vector_index,
|
||||
})
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn struct_to_json(s: &Struct) -> serde_json::Value {
|
||||
@@ -408,7 +230,7 @@ fn json_to_value(v: serde_json::Value) -> Value {
|
||||
Value { kind: Some(kind) }
|
||||
}
|
||||
|
||||
pub(crate) fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
|
||||
fn json_to_struct(map: serde_json::Map<String, serde_json::Value>) -> Struct {
|
||||
Struct {
|
||||
fields: map
|
||||
.into_iter()
|
||||
@@ -706,8 +528,6 @@ pub struct VectorServer {
|
||||
lance_cache_bytes: (usize, usize),
|
||||
leases: SharedLeases,
|
||||
nprobes: Option<usize>,
|
||||
exact_match_limit: Option<usize>,
|
||||
filter_stats: crate::filter_stats::FilterStats,
|
||||
/// URI schemes a table's location may have.
|
||||
schemes: &'static [&'static str],
|
||||
}
|
||||
@@ -731,8 +551,6 @@ impl VectorServer {
|
||||
handles: moka::sync::Cache::new(MAX_HANDLES),
|
||||
leases: SharedLeases::default(),
|
||||
nprobes: None,
|
||||
exact_match_limit: None,
|
||||
filter_stats: Default::default(),
|
||||
schemes: &["s3"],
|
||||
}
|
||||
}
|
||||
@@ -750,13 +568,6 @@ impl VectorServer {
|
||||
self
|
||||
}
|
||||
|
||||
/// Matches of a filter's indexed metadata keys scored exactly instead of
|
||||
/// searched through the vector index, at most; None keeps the default.
|
||||
pub fn with_exact_match_limit(mut self, limit: Option<usize>) -> Self {
|
||||
self.exact_match_limit = limit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Keep tables on the filers and volume servers, not through the gateway.
|
||||
/// Unset only for tests on an in-process store.
|
||||
pub fn with_storage(mut self, storage: DirectReads) -> Self {
|
||||
@@ -929,11 +740,7 @@ impl VectorService for VectorServer {
|
||||
)
|
||||
.await;
|
||||
let dataset = match dataset {
|
||||
Ok(dataset) => {
|
||||
self.filter_stats
|
||||
.forget(&table.id.join("/"), table_id(&dataset));
|
||||
dataset
|
||||
}
|
||||
Ok(dataset) => dataset,
|
||||
Err(error) => match self.open(&table).await {
|
||||
Ok(existing)
|
||||
if matches_index_schema(
|
||||
@@ -984,7 +791,6 @@ impl VectorService for VectorServer {
|
||||
store.remove_dir_all(path).await.map_err(internal)?;
|
||||
}
|
||||
}
|
||||
self.filter_stats.forget(&table.id.join("/"), "");
|
||||
Ok(Response::new(pb::DropIndexResponse {}))
|
||||
}
|
||||
|
||||
@@ -996,37 +802,35 @@ impl VectorService for VectorServer {
|
||||
let table = req
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
if table.version != 0 {
|
||||
return Err(Status::invalid_argument("a write takes no table version"));
|
||||
let dataset = self.open(&table).await?;
|
||||
if req.vectors.is_empty() {
|
||||
return Ok(Response::new(pb::PutVectorsResponse {
|
||||
version: dataset.version().version,
|
||||
}));
|
||||
}
|
||||
// A write commits within its deadline of opening the table or not
|
||||
// at all: a key's column add waits longer than that before the key
|
||||
// is ready, so a write that opened before the add has committed,
|
||||
// and been derived again, before any query uses the key.
|
||||
let merged = tokio::time::timeout(metadata_index::WRITE_DEADLINE, async {
|
||||
let dataset = self.open(&table).await?;
|
||||
if req.vectors.is_empty() {
|
||||
return Ok::<_, Status>((dataset, None));
|
||||
}
|
||||
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
|
||||
Ok((merge_vectors(dataset, &req.vectors).await?, Some(written)))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
Status::deadline_exceeded(format!(
|
||||
"PutVectors took longer than {:?} to commit",
|
||||
metadata_index::WRITE_DEADLINE
|
||||
))
|
||||
})??;
|
||||
let dataset = match merged {
|
||||
(dataset, Some(written)) => {
|
||||
let keys: Vec<String> = req.vectors.iter().map(|v| v.key.clone()).collect();
|
||||
heal_columns(dataset, &written, &keys)
|
||||
.await
|
||||
.map_err(internal)?
|
||||
}
|
||||
(dataset, None) => dataset,
|
||||
};
|
||||
let dimension = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN))
|
||||
.map_err(invalid)?
|
||||
.dimension;
|
||||
let batch = batch_of(
|
||||
&req.vectors,
|
||||
dimension,
|
||||
Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema())),
|
||||
)
|
||||
.map_err(invalid)?;
|
||||
// Upsert by key in one merge commit: no window where the old rows are
|
||||
// already deleted, and a losing writer fails the request instead of
|
||||
// leaving duplicate keys.
|
||||
let mut builder =
|
||||
MergeInsertBuilder::try_new(Arc::new(dataset), vec![KEY_COLUMN.to_string()])
|
||||
.map_err(internal)?;
|
||||
builder.when_matched(WhenMatched::UpdateAll);
|
||||
builder.when_not_matched(WhenNotMatched::InsertAll);
|
||||
let (dataset, _stats) = builder
|
||||
.try_build()
|
||||
.map_err(internal)?
|
||||
.execute_batches(vec![batch])
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
Ok(Response::new(pb::PutVectorsResponse {
|
||||
version: dataset.version().version,
|
||||
}))
|
||||
@@ -1104,9 +908,14 @@ impl VectorService for VectorServer {
|
||||
0 => DEFAULT_PAGE,
|
||||
n => n.min(MAX_PAGE),
|
||||
};
|
||||
// A purge keeps only the latest version, so lease the one this page reads.
|
||||
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
|
||||
if segments > 1 {
|
||||
// Data and metadata come from a second read of this version, so
|
||||
// lease it as get_vectors does.
|
||||
let _lease = if columns.len() > 1 {
|
||||
self.leases.try_pin(&dataset).await.map_err(internal)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (vectors, next) = list_segment(
|
||||
&dataset,
|
||||
&columns,
|
||||
@@ -1144,7 +953,6 @@ impl VectorService for VectorServer {
|
||||
&self,
|
||||
request: Request<pb::QueryVectorsRequest>,
|
||||
) -> Result<Response<pb::QueryVectorsResponse>, Status> {
|
||||
let started = std::time::Instant::now();
|
||||
let req = request.into_inner();
|
||||
let table = req
|
||||
.table
|
||||
@@ -1154,11 +962,6 @@ impl VectorService for VectorServer {
|
||||
// while this query runs against it.
|
||||
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
|
||||
let resolved = VectorTable::resolve(&dataset, None, Some(KEY_COLUMN)).map_err(invalid)?;
|
||||
let filter_keys = if req.filter.is_empty() {
|
||||
Default::default()
|
||||
} else {
|
||||
indexed_filter::filter_keys(&dataset, &req.filter).unwrap_or_default()
|
||||
};
|
||||
if req.query_vector.len() != resolved.dimension {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"query vector carries {} floats, the index dimension is {}",
|
||||
@@ -1183,7 +986,6 @@ impl VectorService for VectorServer {
|
||||
nprobes: self.nprobes,
|
||||
index_segments: None,
|
||||
fragments: None,
|
||||
exact_match_limit: self.exact_match_limit,
|
||||
};
|
||||
let hits = query::ann(&dataset, &resolved, &query)
|
||||
.await
|
||||
@@ -1259,59 +1061,8 @@ impl VectorService for VectorServer {
|
||||
data: data_map.remove(&hit.key).unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
let index = table.id.join("/");
|
||||
self.filter_stats
|
||||
.record(&index, table_id(&dataset), &filter_keys, started.elapsed());
|
||||
Ok(Response::new(pb::QueryVectorsResponse { vectors }))
|
||||
}
|
||||
|
||||
async fn metadata_keys(
|
||||
&self,
|
||||
request: Request<pb::MetadataKeysRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let table = request
|
||||
.into_inner()
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
let dataset = self.open_to_read(&table).await?;
|
||||
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
|
||||
Ok(Response::new(
|
||||
metadata_keys(&dataset).await.map_err(internal)?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_metadata_key(
|
||||
&self,
|
||||
request: Request<pb::UpdateMetadataKeyRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let table = req
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
let mut dataset = self.open(&table).await?;
|
||||
let updated = if req.remove {
|
||||
metadata_index::request_removal(&mut dataset, &req.key).await
|
||||
} else {
|
||||
metadata_index::request(&mut dataset, &req.key).await
|
||||
};
|
||||
updated.map_err(|e| Status::failed_precondition(format!("{e:#}")))?;
|
||||
Ok(Response::new(
|
||||
metadata_keys(&dataset).await.map_err(internal)?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn filter_stats(
|
||||
&self,
|
||||
request: Request<pb::FilterStatsRequest>,
|
||||
) -> Result<Response<pb::FilterStatsResponse>, Status> {
|
||||
let table = request
|
||||
.into_inner()
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
Ok(Response::new(
|
||||
self.filter_stats.snapshot(&table.id.join("/")),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -147,7 +147,6 @@ service VolumeServer {
|
||||
message BatchDeleteRequest {
|
||||
repeated string file_ids = 1;
|
||||
bool skip_cookie_check = 2;
|
||||
bool fsync = 3;
|
||||
}
|
||||
|
||||
message BatchDeleteResponse {
|
||||
|
||||
@@ -306,8 +306,6 @@ pub struct VolumeServerConfig {
|
||||
pub ui_enabled: bool,
|
||||
pub jwt_signing_key: Vec<u8>,
|
||||
pub jwt_signing_expires_seconds: i64,
|
||||
pub jwt_previous_signing_key: Vec<u8>,
|
||||
pub jwt_previous_read_signing_key: Vec<u8>,
|
||||
pub jwt_read_signing_key: Vec<u8>,
|
||||
pub jwt_read_signing_expires_seconds: i64,
|
||||
pub https_cert_file: String,
|
||||
@@ -1001,8 +999,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
|
||||
ui_enabled: sec.jwt_signing_key.is_empty() || sec.access_ui,
|
||||
jwt_signing_key: sec.jwt_signing_key,
|
||||
jwt_signing_expires_seconds: sec.jwt_signing_expires,
|
||||
jwt_previous_signing_key: sec.jwt_previous_signing_key,
|
||||
jwt_previous_read_signing_key: sec.jwt_previous_read_signing_key,
|
||||
jwt_read_signing_key: sec.jwt_read_signing_key,
|
||||
jwt_read_signing_expires_seconds: sec.jwt_read_signing_expires,
|
||||
https_cert_file: sec.https_cert_file,
|
||||
@@ -1034,8 +1030,6 @@ fn default_volume_dir() -> String {
|
||||
pub struct SecurityConfig {
|
||||
pub jwt_signing_key: Vec<u8>,
|
||||
pub jwt_signing_expires: i64,
|
||||
pub jwt_previous_signing_key: Vec<u8>,
|
||||
pub jwt_previous_read_signing_key: Vec<u8>,
|
||||
pub jwt_read_signing_key: Vec<u8>,
|
||||
pub jwt_read_signing_expires: i64,
|
||||
pub https_cert_file: String,
|
||||
@@ -1054,8 +1048,6 @@ pub struct SecurityConfig {
|
||||
pub grpc_volume_allowed_common_names: Vec<String>,
|
||||
pub tls_policy: TlsPolicy,
|
||||
pub access_ui: bool,
|
||||
/// The file was read; otherwise only defaults and WEED_ overrides are set.
|
||||
pub file_read: bool,
|
||||
/// IPs from [guard] white_list in security.toml
|
||||
pub guard_white_list: Vec<String>,
|
||||
}
|
||||
@@ -1188,7 +1180,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
|
||||
match section {
|
||||
Section::JwtSigningRead => match key {
|
||||
"key" => cfg.jwt_read_signing_key = value.as_bytes().to_vec(),
|
||||
"previous_key" => cfg.jwt_previous_read_signing_key = value.as_bytes().to_vec(),
|
||||
"expires_after_seconds" => {
|
||||
cfg.jwt_read_signing_expires = value.parse().unwrap_or(60)
|
||||
}
|
||||
@@ -1196,7 +1187,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
|
||||
},
|
||||
Section::JwtSigning => match key {
|
||||
"key" => cfg.jwt_signing_key = value.as_bytes().to_vec(),
|
||||
"previous_key" => cfg.jwt_previous_signing_key = value.as_bytes().to_vec(),
|
||||
"expires_after_seconds" => {
|
||||
cfg.jwt_signing_expires = value.parse().unwrap_or(10)
|
||||
}
|
||||
@@ -1272,7 +1262,6 @@ fn parse_security_config_with_env(path: &str, env: EnvLookup<'_>) -> SecurityCon
|
||||
// prefix WEED_, uppercase, replace . with _).
|
||||
// e.g. WEED_JWT_SIGNING_KEY overrides [jwt.signing] key
|
||||
apply_env_overrides(&mut cfg, env);
|
||||
cfg.file_read = true;
|
||||
|
||||
cfg
|
||||
}
|
||||
@@ -1311,12 +1300,6 @@ fn apply_env_overrides(cfg: &mut SecurityConfig, env: EnvLookup<'_>) {
|
||||
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_EXPIRES_AFTER_SECONDS") {
|
||||
cfg.jwt_signing_expires = v.parse().unwrap_or(cfg.jwt_signing_expires);
|
||||
}
|
||||
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_PREVIOUS_KEY") {
|
||||
cfg.jwt_previous_signing_key = v.into_bytes();
|
||||
}
|
||||
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_READ_PREVIOUS_KEY") {
|
||||
cfg.jwt_previous_read_signing_key = v.into_bytes();
|
||||
}
|
||||
if let Some(v) = env_string(env, "WEED_JWT_SIGNING_READ_KEY") {
|
||||
cfg.jwt_read_signing_key = v.into_bytes();
|
||||
}
|
||||
@@ -2194,50 +2177,6 @@ cipher_suites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
|
||||
assert_eq!(cfg.jwt_signing_key, b"env-secret");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_previous_signing_keys_from_file_and_env() {
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(
|
||||
tmp.path(),
|
||||
r#"
|
||||
[jwt.signing]
|
||||
key = "write-new"
|
||||
previous_key = "write-old"
|
||||
[jwt.signing.read]
|
||||
key = "read-new"
|
||||
previous_key = "read-old"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let path = tmp.path().to_str().unwrap();
|
||||
|
||||
let cfg = parse_security_config_with_env(path, &fake_env(&[]));
|
||||
assert_eq!(cfg.jwt_previous_signing_key, b"write-old");
|
||||
assert_eq!(cfg.jwt_previous_read_signing_key, b"read-old");
|
||||
|
||||
let env = fake_env(&[
|
||||
("WEED_JWT_SIGNING_PREVIOUS_KEY", "env-write-old"),
|
||||
("WEED_JWT_SIGNING_READ_PREVIOUS_KEY", "env-read-old"),
|
||||
]);
|
||||
let cfg = parse_security_config_with_env(path, &env);
|
||||
assert_eq!(cfg.jwt_previous_signing_key, b"env-write-old");
|
||||
assert_eq!(cfg.jwt_previous_read_signing_key, b"env-read-old");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_security_config_file_read() {
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(tmp.path(), "[jwt.signing]\nkey = \"secret\"\n").unwrap();
|
||||
let path = tmp.path().to_str().unwrap().to_string();
|
||||
assert!(parse_security_config_with_env(&path, &fake_env(&[])).file_read);
|
||||
|
||||
drop(tmp);
|
||||
let env = fake_env(&[("WEED_JWT_SIGNING_KEY", "env-secret")]);
|
||||
let cfg = parse_security_config_with_env(&path, &env);
|
||||
assert!(!cfg.file_read);
|
||||
assert_eq!(cfg.jwt_signing_key, b"env-secret");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_env_override_takes_precedence_over_file() {
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
|
||||
@@ -340,7 +340,7 @@ async fn run(
|
||||
store.set_volume_load_guard(volume_copies_in_flight.clone());
|
||||
|
||||
// Build shared state
|
||||
let mut guard = Guard::new(
|
||||
let guard = Guard::new(
|
||||
&config.white_list,
|
||||
SigningKey(config.jwt_signing_key.clone()),
|
||||
config.jwt_signing_expires_seconds,
|
||||
@@ -358,8 +358,6 @@ async fn run(
|
||||
|
||||
let security_file = config.security_file.clone();
|
||||
let cli_white_list = config.white_list.clone();
|
||||
guard.previous_signing_key = SigningKey(config.jwt_previous_signing_key.clone());
|
||||
guard.previous_read_signing_key = SigningKey(config.jwt_previous_read_signing_key.clone());
|
||||
|
||||
let state = Arc::new(VolumeServerState {
|
||||
store: RwLock::new(store),
|
||||
@@ -418,7 +416,6 @@ async fn run(
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
},
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight,
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
@@ -621,13 +618,9 @@ async fn run(
|
||||
}
|
||||
|
||||
// 2. Reload security config (Go's Reload)
|
||||
'security: {
|
||||
{
|
||||
info!("Reloading security config...");
|
||||
let sec = config::parse_security_config(&state_reload.security_file);
|
||||
if !sec.file_read {
|
||||
warn!("SIGHUP: security config cannot be read, keeping the keys in effect");
|
||||
break 'security;
|
||||
}
|
||||
let mut whitelist = state_reload.cli_white_list.clone();
|
||||
whitelist.extend(sec.guard_white_list.iter().cloned());
|
||||
let mut guard = state_reload.guard.write().unwrap();
|
||||
@@ -637,8 +630,6 @@ async fn run(
|
||||
sec.jwt_signing_expires,
|
||||
SigningKey(sec.jwt_read_signing_key),
|
||||
sec.jwt_read_signing_expires,
|
||||
SigningKey(sec.jwt_previous_signing_key),
|
||||
SigningKey(sec.jwt_previous_read_signing_key),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -85,19 +85,6 @@ pub static VACUUMING_COMMIT_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(||
|
||||
.expect("metric can be created")
|
||||
});
|
||||
|
||||
/// Live needles vacuum dropped because their data could not be read back
|
||||
/// intact, with label `collection`.
|
||||
pub static VACUUMING_DROPPED_NEEDLES_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
|
||||
IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_vacuuming_dropped_needles",
|
||||
"Counter of live needles dropped by vacuum because their data could not be read back intact.",
|
||||
),
|
||||
&["collection"],
|
||||
)
|
||||
.expect("metric can be created")
|
||||
});
|
||||
|
||||
/// Vacuuming duration histogram with label `type` (compact/commit).
|
||||
pub static VACUUMING_HISTOGRAM: LazyLock<HistogramVec> = LazyLock::new(|| {
|
||||
HistogramVec::new(
|
||||
@@ -415,7 +402,6 @@ pub fn register_metrics() {
|
||||
Box::new(VACUUMING_COMPACT_COUNTER.clone()),
|
||||
Box::new(VACUUMING_COMMIT_COUNTER.clone()),
|
||||
Box::new(VACUUMING_HISTOGRAM.clone()),
|
||||
Box::new(VACUUMING_DROPPED_NEEDLES_COUNTER.clone()),
|
||||
Box::new(VOLUME_GAUGE.clone()),
|
||||
Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
|
||||
Box::new(MAX_VOLUMES.clone()),
|
||||
|
||||
+23
-207
@@ -83,13 +83,7 @@ pub fn gen_jwt(
|
||||
}
|
||||
|
||||
/// Decode and validate a JWT token.
|
||||
/// A token whose signature does not match `signing_key` is checked against
|
||||
/// each non-empty `previous_keys` entry, like Go's `security.DecodeJwt`.
|
||||
pub fn decode_jwt(
|
||||
signing_key: &SigningKey,
|
||||
token: &str,
|
||||
previous_keys: &[&SigningKey],
|
||||
) -> Result<FileIdClaims, JwtError> {
|
||||
pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims, JwtError> {
|
||||
if signing_key.is_empty() {
|
||||
return Err(JwtError::NoSigningKey);
|
||||
}
|
||||
@@ -103,64 +97,13 @@ pub fn decode_jwt(
|
||||
validation.validate_nbf = true;
|
||||
validation.leeway = 0;
|
||||
|
||||
let mut result = decode::<FileIdClaims>(
|
||||
let data = decode::<FileIdClaims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(&signing_key.0),
|
||||
&validation,
|
||||
);
|
||||
for previous in previous_keys {
|
||||
let bad_signature = matches!(&result, Err(e)
|
||||
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::InvalidSignature));
|
||||
if !bad_signature {
|
||||
break;
|
||||
}
|
||||
if previous.is_empty() || previous.0 == signing_key.0 {
|
||||
continue;
|
||||
}
|
||||
result = decode::<FileIdClaims>(token, &DecodingKey::from_secret(&previous.0), &validation);
|
||||
}
|
||||
)?;
|
||||
|
||||
Ok(result?.claims)
|
||||
}
|
||||
|
||||
/// Reports a signed, fid-bound write token whose only failure is expiry. It
|
||||
/// never authorizes a request. Mirrors Go's `security.IsExpiredVolumeJwt`.
|
||||
pub fn is_expired_volume_jwt(
|
||||
signing_key: &SigningKey,
|
||||
token: &str,
|
||||
expected_fid: &str,
|
||||
previous_keys: &[&SigningKey],
|
||||
) -> bool {
|
||||
match decode_jwt(signing_key, token, previous_keys) {
|
||||
Err(JwtError::Jwt(e))
|
||||
if matches!(e.kind(), jsonwebtoken::errors::ErrorKind::ExpiredSignature) => {}
|
||||
_ => return false,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ExpiredClaims {
|
||||
fid: Option<String>,
|
||||
iat: Option<u64>,
|
||||
}
|
||||
let mut validation = Validation::new(Algorithm::HS256);
|
||||
validation.required_spec_claims.clear();
|
||||
validation.validate_exp = false;
|
||||
validation.validate_nbf = true;
|
||||
validation.leeway = 0;
|
||||
let Some(data) = std::iter::once(signing_key)
|
||||
.chain(previous_keys.iter().copied())
|
||||
.filter(|key| !key.is_empty())
|
||||
.find_map(|key| {
|
||||
decode::<ExpiredClaims>(token, &DecodingKey::from_secret(&key.0), &validation).ok()
|
||||
})
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
data.claims.fid.as_deref() == Some(expected_fid) && data.claims.iat.is_none_or(|iat| iat <= now)
|
||||
Ok(data.claims)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -175,9 +118,6 @@ pub struct Guard {
|
||||
pub expires_after_sec: i64,
|
||||
pub read_signing_key: SigningKey,
|
||||
pub read_expires_after_sec: i64,
|
||||
/// Accepted when verifying, never used to sign.
|
||||
pub previous_signing_key: SigningKey,
|
||||
pub previous_read_signing_key: SigningKey,
|
||||
/// Combined flag: true when whitelist is non-empty OR signing key is present.
|
||||
/// Matches Go's `isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0`.
|
||||
is_write_active: bool,
|
||||
@@ -198,8 +138,6 @@ impl Guard {
|
||||
expires_after_sec,
|
||||
read_signing_key,
|
||||
read_expires_after_sec,
|
||||
previous_signing_key: SigningKey(vec![]),
|
||||
previous_read_signing_key: SigningKey(vec![]),
|
||||
is_write_active: false,
|
||||
};
|
||||
guard.update_whitelist(whitelist);
|
||||
@@ -244,15 +182,11 @@ impl Guard {
|
||||
expires_after_sec: i64,
|
||||
read_signing_key: SigningKey,
|
||||
read_expires_after_sec: i64,
|
||||
previous_signing_key: SigningKey,
|
||||
previous_read_signing_key: SigningKey,
|
||||
) {
|
||||
self.signing_key = signing_key;
|
||||
self.expires_after_sec = expires_after_sec;
|
||||
self.read_signing_key = read_signing_key;
|
||||
self.read_expires_after_sec = read_expires_after_sec;
|
||||
self.previous_signing_key = previous_signing_key;
|
||||
self.previous_read_signing_key = previous_read_signing_key;
|
||||
|
||||
let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
|
||||
self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
|
||||
@@ -295,16 +229,6 @@ impl Guard {
|
||||
false
|
||||
}
|
||||
|
||||
/// The write (or read) signing key and the previous key still accepted for
|
||||
/// it. Mirrors Go's `Guard.VerificationKeys`.
|
||||
pub fn verification_keys(&self, is_write: bool) -> (&SigningKey, &SigningKey) {
|
||||
if is_write {
|
||||
(&self.signing_key, &self.previous_signing_key)
|
||||
} else {
|
||||
(&self.read_signing_key, &self.previous_read_signing_key)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a read signing key is configured.
|
||||
pub fn has_read_signing_key(&self) -> bool {
|
||||
!self.read_signing_key.is_empty()
|
||||
@@ -314,14 +238,18 @@ impl Guard {
|
||||
/// `is_write` determines which signing key to use.
|
||||
/// Returns Ok(()) if valid, or if security is disabled.
|
||||
pub fn check_jwt(&self, token: Option<&str>, is_write: bool) -> Result<(), JwtError> {
|
||||
let (key, previous) = self.verification_keys(is_write);
|
||||
let key = if is_write {
|
||||
&self.signing_key
|
||||
} else {
|
||||
&self.read_signing_key
|
||||
};
|
||||
|
||||
if key.is_empty() {
|
||||
return Ok(()); // Security disabled for this operation type
|
||||
}
|
||||
|
||||
let token = token.ok_or(JwtError::MissingToken)?;
|
||||
decode_jwt(key, token, &[previous])?;
|
||||
decode_jwt(key, token)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -332,14 +260,18 @@ impl Guard {
|
||||
expected_fid: &str,
|
||||
is_write: bool,
|
||||
) -> Result<(), JwtError> {
|
||||
let (key, previous) = self.verification_keys(is_write);
|
||||
let key = if is_write {
|
||||
&self.signing_key
|
||||
} else {
|
||||
&self.read_signing_key
|
||||
};
|
||||
|
||||
if key.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let token = token.ok_or(JwtError::MissingToken)?;
|
||||
let claims = decode_jwt(key, token, &[previous])?;
|
||||
let claims = decode_jwt(key, token)?;
|
||||
|
||||
match claims.fid {
|
||||
None => {
|
||||
@@ -453,7 +385,7 @@ mod tests {
|
||||
fn test_jwt_round_trip() {
|
||||
let key = SigningKey::from_string("test-secret-key");
|
||||
let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
|
||||
let claims = decode_jwt(&key, &token, &[]).unwrap();
|
||||
let claims = decode_jwt(&key, &token).unwrap();
|
||||
assert_eq!(claims.fid, Some("3,01637037d6".to_string()));
|
||||
}
|
||||
|
||||
@@ -466,7 +398,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_jwt_invalid_token() {
|
||||
let key = SigningKey::from_string("secret");
|
||||
let result = decode_jwt(&key, "invalid.token.here", &[]);
|
||||
let result = decode_jwt(&key, "invalid.token.here");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -475,60 +407,7 @@ mod tests {
|
||||
let key1 = SigningKey::from_string("secret1");
|
||||
let key2 = SigningKey::from_string("secret2");
|
||||
let token = gen_jwt(&key1, 3600, "1,abc").unwrap();
|
||||
assert!(decode_jwt(&key2, &token, &[]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_expired_volume_jwt_classifies_expiry_only() {
|
||||
let key = SigningKey::from_string("write-key");
|
||||
let fid = "3,01637037d6";
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let sign = |claims: serde_json::Value, key: &SigningKey| {
|
||||
encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&claims,
|
||||
&EncodingKey::from_secret(&key.0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let expired = sign(serde_json::json!({"fid": fid, "exp": now - 10}), &key);
|
||||
assert!(is_expired_volume_jwt(&key, &expired, fid, &[]));
|
||||
assert!(!is_expired_volume_jwt(&key, &expired, "3,01637037d7", &[]));
|
||||
assert!(!is_expired_volume_jwt(
|
||||
&SigningKey::from_string("other-key"),
|
||||
&expired,
|
||||
fid,
|
||||
&[]
|
||||
));
|
||||
assert!(!is_expired_volume_jwt(
|
||||
&SigningKey(vec![]),
|
||||
&expired,
|
||||
fid,
|
||||
&[]
|
||||
));
|
||||
let immature = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "nbf": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &immature, fid, &[]));
|
||||
let future_issued = sign(
|
||||
serde_json::json!({"fid": fid, "exp": now - 10, "iat": now + 60}),
|
||||
&key,
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &future_issued, fid, &[]));
|
||||
let live = sign(serde_json::json!({"fid": fid, "exp": now + 60}), &key);
|
||||
assert!(!is_expired_volume_jwt(&key, &live, fid, &[]));
|
||||
let unsigned = format!(
|
||||
"eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.{}.",
|
||||
base64::Engine::encode(
|
||||
&base64::engine::general_purpose::URL_SAFE_NO_PAD,
|
||||
serde_json::json!({"fid": fid, "exp": now - 10}).to_string()
|
||||
)
|
||||
);
|
||||
assert!(!is_expired_volume_jwt(&key, &unsigned, fid, &[]));
|
||||
assert!(decode_jwt(&key2, &token).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -628,88 +507,25 @@ mod tests {
|
||||
11,
|
||||
SigningKey::from_string("new-read"),
|
||||
61,
|
||||
SigningKey(vec![]),
|
||||
SigningKey(vec![]),
|
||||
);
|
||||
|
||||
let token = gen_jwt(&guard.signing_key, 60, "3,01637037d6").unwrap();
|
||||
// Validates with the rotated key, not the old one.
|
||||
assert!(decode_jwt(&guard.signing_key, &token, &[]).is_ok());
|
||||
assert!(decode_jwt(&SigningKey::from_string("old-write"), &token, &[]).is_err());
|
||||
assert!(decode_jwt(&guard.signing_key, &token).is_ok());
|
||||
assert!(decode_jwt(&SigningKey::from_string("old-write"), &token).is_err());
|
||||
assert_eq!(guard.expires_after_sec, 11);
|
||||
assert_eq!(guard.read_expires_after_sec, 61);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_previous_signing_key_is_accepted() {
|
||||
let fid = "3,01637037d6";
|
||||
let accepts = |guard: &Guard, key: &str, is_write: bool| {
|
||||
let token = gen_jwt(&SigningKey::from_string(key), 60, fid).unwrap();
|
||||
guard
|
||||
.check_jwt_for_file(Some(&token), fid, is_write)
|
||||
.is_ok()
|
||||
};
|
||||
let mut guard = Guard::new(
|
||||
&[],
|
||||
SigningKey::from_string("write-new"),
|
||||
10,
|
||||
SigningKey::from_string("read-new"),
|
||||
60,
|
||||
);
|
||||
guard.previous_signing_key = SigningKey::from_string("write-old");
|
||||
guard.previous_read_signing_key = SigningKey::from_string("read-old");
|
||||
|
||||
assert!(accepts(&guard, "write-new", true));
|
||||
assert!(accepts(&guard, "write-old", true));
|
||||
assert!(accepts(&guard, "read-old", false));
|
||||
assert!(!accepts(&guard, "other", true));
|
||||
assert!(!accepts(&guard, "read-old", true));
|
||||
|
||||
let expired = encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&serde_json::json!({"fid": fid, "exp": 1}),
|
||||
&EncodingKey::from_secret(b"write-old"),
|
||||
)
|
||||
.unwrap();
|
||||
let (key, previous) = guard.verification_keys(true);
|
||||
assert!(is_expired_volume_jwt(key, &expired, fid, &[previous]));
|
||||
assert!(!is_expired_volume_jwt(key, &expired, fid, &[]));
|
||||
|
||||
guard.update_signing_keys(
|
||||
SigningKey::from_string("write-new"),
|
||||
10,
|
||||
SigningKey::from_string("read-new"),
|
||||
60,
|
||||
SigningKey(vec![]),
|
||||
SigningKey(vec![]),
|
||||
);
|
||||
assert!(!accepts(&guard, "write-old", true));
|
||||
assert!(!accepts(&guard, "read-old", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_update_signing_keys_toggles_write_active() {
|
||||
let mut guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
assert!(guard.check_jwt(None, true).is_ok());
|
||||
|
||||
guard.update_signing_keys(
|
||||
SigningKey::from_string("write"),
|
||||
10,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
SigningKey(vec![]),
|
||||
);
|
||||
guard.update_signing_keys(SigningKey::from_string("write"), 10, SigningKey(vec![]), 0);
|
||||
assert!(guard.is_write_active);
|
||||
|
||||
guard.update_signing_keys(
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
SigningKey(vec![]),
|
||||
);
|
||||
guard.update_signing_keys(SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
assert!(!guard.is_write_active);
|
||||
}
|
||||
|
||||
|
||||
@@ -110,17 +110,6 @@ pub fn load_outgoing_grpc_tls(
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl OutgoingGrpcTlsConfig {
|
||||
pub(crate) fn self_signed_for_test() -> Self {
|
||||
Self {
|
||||
cert_pem: tests::TEST_CERT_PEM.to_string(),
|
||||
key_pem: tests::TEST_KEY_PEM.to_string(),
|
||||
ca_pem: tests::TEST_CERT_PEM.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn grpc_endpoint_uri(grpc_host_port: &str, tls: Option<&OutgoingGrpcTlsConfig>) -> String {
|
||||
let scheme = if tls.is_some() { "https" } else { "http" };
|
||||
format!("{}://{}", scheme, grpc_host_port)
|
||||
@@ -371,8 +360,8 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
|
||||
pub(super) const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
|
||||
const TEST_CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\nMIIBPDCB76ADAgECAhRuRPQgeAu43BT/M7EfAWSdapVdYDAFBgMrZXAwFDESMBAG\nA1UEAwwJbG9jYWxob3N0MB4XDTI2MDcwNTE2MTUyOVoXDTM2MDcwMjE2MTUyOVow\nFDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEAr/3bNIFI+8V32oCiY6y+\nXRFmZpdNQ2g//VtRkT+nQg+jUzBRMB0GA1UdDgQWBBTsy9tLf1zPiXCQfgci6zNi\ndEzRSjAfBgNVHSMEGDAWgBTsy9tLf1zPiXCQfgci6zNidEzRSjAPBgNVHRMBAf8E\nBTADAQH/MAUGAytlcANBAIvsdw0IbvOBBkb9cd7BfMJfIP9pQQrAL03pCRWJFnFh\nSysaLVgFXI4T078IiaM874oO+iB+5vNbWEpc7CkGow4=\n-----END CERTIFICATE-----\n";
|
||||
const TEST_KEY_PEM: &str = "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIHbyn71Kk+Y7KT3sBctit7uZpErpoH6qDbFj6P8qGaZH\n-----END PRIVATE KEY-----\n";
|
||||
|
||||
#[test]
|
||||
fn test_build_grpc_endpoint_with_tls_resolves_crypto_provider() {
|
||||
@@ -436,8 +425,6 @@ mod tests {
|
||||
ui_enabled: false,
|
||||
jwt_signing_key: vec![],
|
||||
jwt_signing_expires_seconds: 0,
|
||||
jwt_previous_signing_key: vec![],
|
||||
jwt_previous_read_signing_key: vec![],
|
||||
jwt_read_signing_key: vec![],
|
||||
jwt_read_signing_expires_seconds: 0,
|
||||
https_cert_file: String::new(),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1698,7 +1698,6 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
|
||||
@@ -156,8 +156,6 @@ mod tests {
|
||||
ui_enabled: false,
|
||||
jwt_signing_key: vec![],
|
||||
jwt_signing_expires_seconds: 0,
|
||||
jwt_previous_signing_key: vec![],
|
||||
jwt_previous_read_signing_key: vec![],
|
||||
jwt_read_signing_key: vec![],
|
||||
jwt_read_signing_expires_seconds: 0,
|
||||
https_cert_file: String::new(),
|
||||
|
||||
@@ -2384,45 +2384,23 @@ pub(crate) async fn recover_missing_ec_indexes(
|
||||
"cannot look up peers to recover missing .ecx: {}",
|
||||
e
|
||||
);
|
||||
Vec::new()
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// Under the volume's lifecycle lock, so an in-flight VolumeEcShardsCopy
|
||||
// of it, which may unlink the shards it renamed in, never has them
|
||||
// mounted from under it.
|
||||
let _lifecycle = state.ec_lifecycle_lock(m.vid).lock_owned().await;
|
||||
let current = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.collect_ec_volumes_missing_index_only(Some(m.vid))
|
||||
.into_iter()
|
||||
.find(|x| x.vid == m.vid && x.collection == m.collection);
|
||||
// A copy holding the lock may have installed the index meanwhile: skip
|
||||
// the fetch, but its shards still need the mount.
|
||||
if let Some(current) = ¤t {
|
||||
if peers.is_empty() {
|
||||
tracing::warn!(
|
||||
volume_id = m.vid.0,
|
||||
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !fetch_ec_index_from_peers(state, current, &peers).await {
|
||||
continue;
|
||||
}
|
||||
if peers.is_empty() {
|
||||
tracing::warn!(
|
||||
volume_id = m.vid.0,
|
||||
"shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
state
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.mount_recovered_ec_shards_only(Some(m.vid));
|
||||
if current.is_some() {
|
||||
if fetch_ec_index_from_peers(state, m, &peers).await {
|
||||
recovered += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if recovered > 0 {
|
||||
state.store.write().unwrap().mount_recovered_ec_shards();
|
||||
tracing::info!(
|
||||
"recovered missing EC index for {} volume(s) from peers and mounted their shards",
|
||||
recovered
|
||||
|
||||
@@ -125,11 +125,6 @@ pub struct VolumeServerState {
|
||||
pub cli_white_list: Vec<String>,
|
||||
/// Path to state.pb file for persisting VolumeServerState across restarts.
|
||||
pub state_file_path: String,
|
||||
/// Per-vid EC lifecycle lock, serializing EC ops that install or remove a
|
||||
/// vid's files. A tokio Mutex so the guard survives the copy's `.await` IO.
|
||||
pub ec_lifecycle_locks: std::sync::Mutex<
|
||||
std::collections::HashMap<crate::storage::types::VolumeId, Arc<tokio::sync::Mutex<()>>>,
|
||||
>,
|
||||
/// Volumes with an EC decode in flight. A dropped request leaves the
|
||||
/// blocking job running; this keeps a retry from racing it on the
|
||||
/// same volume files.
|
||||
@@ -156,20 +151,6 @@ pub struct VolumeServerState {
|
||||
}
|
||||
|
||||
impl VolumeServerState {
|
||||
/// Return the per-vid EC lifecycle lock, creating it on first use. Entries
|
||||
/// are never reaped.
|
||||
pub fn ec_lifecycle_lock(
|
||||
&self,
|
||||
vid: crate::storage::types::VolumeId,
|
||||
) -> Arc<tokio::sync::Mutex<()>> {
|
||||
self.ec_lifecycle_locks
|
||||
.lock()
|
||||
.unwrap()
|
||||
.entry(vid)
|
||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// The master lookups should go to: the leader the heartbeat confirmed,
|
||||
/// or empty while none is known, in its announced `host:port[.grpcPort]`
|
||||
/// form. Mirrors Go's `vs.getCurrentMaster()`.
|
||||
|
||||
@@ -467,7 +467,6 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
|
||||
@@ -829,24 +829,6 @@ impl DiskLocation {
|
||||
self.volumes.len()
|
||||
}
|
||||
|
||||
/// Number of local volumes of `collection` that still take writes: not
|
||||
/// read-only and, when `volume_size_limit` is known (non zero), with a .dat
|
||||
/// below it. The .dat size is what the master compares with the limit to
|
||||
/// stop writes. Mirrors Go's WritableCollectionVolumesLen.
|
||||
pub fn writable_collection_volumes_len(
|
||||
&self,
|
||||
collection: &str,
|
||||
volume_size_limit: u64,
|
||||
) -> usize {
|
||||
self.volumes
|
||||
.values()
|
||||
.filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only())
|
||||
.filter(|v| {
|
||||
volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit
|
||||
})
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Get all volume IDs, sorted.
|
||||
pub fn volume_ids(&self) -> Vec<VolumeId> {
|
||||
let mut ids: Vec<VolumeId> = self.volumes.keys().copied().collect();
|
||||
|
||||
@@ -292,20 +292,12 @@ impl Store {
|
||||
|
||||
// ---- Volume lifecycle ----
|
||||
|
||||
/// Find the location with the fewest writable volumes of `collection`, so
|
||||
/// they do not share one disk's write bandwidth, then the most free slots
|
||||
/// (load-balance) of the given disk type. Matches Go's FindFreeLocation:
|
||||
/// accounts for EC shards when computing free slots.
|
||||
fn find_free_location(
|
||||
&self,
|
||||
disk_type: &DiskType,
|
||||
collection: &str,
|
||||
preallocate: u64,
|
||||
) -> Option<usize> {
|
||||
/// Find the location with fewest volumes (load-balance) of the given disk type.
|
||||
/// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots.
|
||||
fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option<usize> {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed);
|
||||
let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free)
|
||||
let mut best: Option<(usize, i64)> = None; // (index, effective_free)
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if &loc.disk_type != disk_type {
|
||||
continue;
|
||||
@@ -335,14 +327,11 @@ impl Store {
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let writable = loc.writable_collection_volumes_len(collection, volume_size_limit);
|
||||
if best.is_none_or(|(_, min_writable, max_free)| {
|
||||
writable < min_writable || (writable == min_writable && effective_free > max_free)
|
||||
}) {
|
||||
best = Some((i, writable, effective_free));
|
||||
if best.is_none() || effective_free > best.unwrap().1 {
|
||||
best = Some((i, effective_free));
|
||||
}
|
||||
}
|
||||
best.map(|(i, _, _)| i)
|
||||
best.map(|(i, _)| i)
|
||||
}
|
||||
|
||||
/// Find a free location matching a predicate.
|
||||
@@ -612,7 +601,7 @@ impl Store {
|
||||
return Err(VolumeError::AlreadyExists);
|
||||
}
|
||||
let loc_idx = self
|
||||
.find_free_location(&disk_type, spec.collection, spec.preallocate)
|
||||
.find_free_location(&disk_type, spec.preallocate)
|
||||
.ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::other(format!(
|
||||
"no free location for disk type {:?}",
|
||||
@@ -1053,18 +1042,13 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Delete a needle from a volume.
|
||||
pub fn delete_volume_needle(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
fsync: bool,
|
||||
) -> Result<Size, VolumeError> {
|
||||
pub fn delete_volume_needle(&self, vid: VolumeId, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
// Match Go's DeleteVolumeNeedle: check noWriteOrDelete before proceeding.
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
if vol.is_no_write_or_delete() {
|
||||
return Err(VolumeError::ReadOnly(vid));
|
||||
}
|
||||
vol.delete_needle(n, fsync)
|
||||
vol.delete_needle(n)
|
||||
}
|
||||
|
||||
// ---- Collection operations ----
|
||||
@@ -2939,9 +2923,7 @@ mod tests {
|
||||
cookie: Cookie(0xaa),
|
||||
..Needle::default()
|
||||
};
|
||||
let deleted = store
|
||||
.delete_volume_needle(VolumeId(1), &mut del_n, false)
|
||||
.unwrap();
|
||||
let deleted = store.delete_volume_needle(VolumeId(1), &mut del_n).unwrap();
|
||||
assert!(deleted.0 > 0);
|
||||
}
|
||||
|
||||
@@ -3050,87 +3032,6 @@ mod tests {
|
||||
assert_eq!(store.locations[1].volumes_len(), 1);
|
||||
}
|
||||
|
||||
fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) {
|
||||
let spec = VolumeSpec {
|
||||
collection,
|
||||
..VolumeSpec::default()
|
||||
};
|
||||
store
|
||||
.add_volume(VolumeId(vid), DiskType::HardDrive, &spec)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn location_of(store: &Store, vid: u32) -> Option<usize> {
|
||||
store
|
||||
.locations
|
||||
.iter()
|
||||
.position(|loc| loc.find_volume(VolumeId(vid)).is_some())
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks.
|
||||
#[test]
|
||||
fn test_add_volume_spreads_collection_over_disks() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "other");
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
let first = location_of(&store, 3);
|
||||
// Leave the disk with the "spread" volume holding fewer volumes overall.
|
||||
let other_on_first = if first == Some(1) { 2 } else { 1 };
|
||||
store
|
||||
.delete_volume(VolumeId(other_on_first), false, false, false)
|
||||
.unwrap();
|
||||
|
||||
add_spread_test_volume(&mut store, 4, "spread");
|
||||
assert_ne!(location_of(&store, 4), first);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection.
|
||||
#[test]
|
||||
fn test_add_volume_ignores_full_volumes_of_collection() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
// Every .dat holds at least its superblock, so all volumes are full.
|
||||
store.volume_size_limit.store(1, Ordering::Relaxed);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "spread");
|
||||
let full_disk = location_of(&store, 1);
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
assert_eq!(location_of(&store, 3), full_disk);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection.
|
||||
#[test]
|
||||
fn test_add_volume_ignores_read_only_volumes_of_collection() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let mut store =
|
||||
make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]);
|
||||
|
||||
add_spread_test_volume(&mut store, 1, "spread");
|
||||
let writable_disk = location_of(&store, 1);
|
||||
add_spread_test_volume(&mut store, 2, "other");
|
||||
add_spread_test_volume(&mut store, 3, "spread");
|
||||
assert_ne!(location_of(&store, 3), writable_disk);
|
||||
store
|
||||
.find_volume(VolumeId(3))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_read_only()
|
||||
.unwrap();
|
||||
|
||||
// The other disk holds more volumes, but its "spread" volume is read-only.
|
||||
add_spread_test_volume(&mut store, 4, "spread");
|
||||
assert_ne!(location_of(&store, 4), writable_disk);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_store_delete_collection() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -3499,7 +3400,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
|
||||
let size_before = store.find_volume(vid).unwrap().1.dat_file_size().unwrap();
|
||||
|
||||
@@ -3823,7 +3724,7 @@ mod tests {
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
let revision = {
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
v.super_block().compaction_revision
|
||||
@@ -3848,7 +3749,7 @@ mod tests {
|
||||
cookie: Cookie(4),
|
||||
..Needle::default()
|
||||
};
|
||||
store.delete_volume_needle(vid, &mut del, false).unwrap();
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
|
||||
drop(release_tx);
|
||||
copy.join().unwrap().unwrap();
|
||||
|
||||
@@ -36,14 +36,10 @@ impl Store {
|
||||
/// `reconcile_ec_shards_across_disks` so the orphan pass can
|
||||
/// prefer the local idx_directory.
|
||||
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
|
||||
self.mirror_ec_metadata_to_shard_disks_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn mirror_ec_metadata_to_shard_disks_only(&mut self, only: Option<VolumeId>) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
let owners = self.index_ecx_owners_for_mirror(only);
|
||||
let owners = self.index_ecx_owners_for_mirror();
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -58,7 +54,7 @@ impl Store {
|
||||
}
|
||||
let mut mirrors: Vec<Mirror> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_shard_disk_volumes(loc, only);
|
||||
let orphans = collect_shard_disk_volumes(loc);
|
||||
for (key, _shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
continue;
|
||||
@@ -117,7 +113,7 @@ impl Store {
|
||||
// Records both idx_dir (where .ecx was found) and data_dir, so
|
||||
// the mirror can resolve .vif from data_dir even when .ecx lives
|
||||
// in idx_directory.
|
||||
fn index_ecx_owners_for_mirror(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwner> {
|
||||
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
|
||||
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
@@ -137,18 +133,15 @@ impl Store {
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
|
||||
// credible owner — skip it so the scan keeps looking for a
|
||||
// real index on a sibling disk (Go's indexEcxOwners).
|
||||
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
owners
|
||||
.entry(EcKey { collection, vid })
|
||||
.or_insert_with(|| EcxOwner {
|
||||
@@ -304,10 +297,7 @@ fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_shard_disk_volumes(
|
||||
loc: &DiskLocation,
|
||||
only: Option<VolumeId>,
|
||||
) -> HashMap<EcKey, Vec<String>> {
|
||||
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
|
||||
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
return out;
|
||||
@@ -324,16 +314,13 @@ fn collect_shard_disk_volumes(
|
||||
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
match ent.metadata() {
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
out.entry(EcKey { collection, vid }).or_default().push(name);
|
||||
}
|
||||
out
|
||||
@@ -701,7 +688,7 @@ mod tests {
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
let owners = store.index_ecx_owners_for_mirror(None);
|
||||
let owners = store.index_ecx_owners_for_mirror();
|
||||
let owner = owners
|
||||
.get(&EcKey {
|
||||
collection: collection.to_string(),
|
||||
|
||||
@@ -92,15 +92,11 @@ impl Store {
|
||||
///
|
||||
/// Mirrors `Store.reconcileEcShardsAcrossDisks` in Go.
|
||||
pub fn reconcile_ec_shards_across_disks(&mut self) {
|
||||
self.reconcile_ec_shards_across_disks_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn reconcile_ec_shards_across_disks_only(&mut self, only: Option<VolumeId>) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
|
||||
let owners = self.index_ecx_owners(only);
|
||||
let owners = self.index_ecx_owners();
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -110,7 +106,7 @@ impl Store {
|
||||
// loc.idx_directory instead of the owner disk.
|
||||
let mut to_load: Vec<OrphanShardLoad> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_orphan_ec_shards(loc, loc_idx, only);
|
||||
let orphans = collect_orphan_ec_shards(loc, loc_idx);
|
||||
for (key, shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
warn!(
|
||||
@@ -406,7 +402,7 @@ impl Store {
|
||||
/// matched so downstream loaders point `EcVolume::new` at the
|
||||
/// directory that really has the file. The first owner found wins;
|
||||
/// duplicates across disks are unusual but tolerated.
|
||||
fn index_ecx_owners(&self, only: Option<VolumeId>) -> HashMap<EcKey, EcxOwnerInfo> {
|
||||
fn index_ecx_owners(&self) -> HashMap<EcKey, EcxOwnerInfo> {
|
||||
let mut owners: HashMap<EcKey, EcxOwnerInfo> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
@@ -426,18 +422,15 @@ impl Store {
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// A 0-byte .ecx is a corrupt stub from a failed copy, not a
|
||||
// credible owner — skip it so the scan keeps looking for a
|
||||
// real index on a sibling disk (Go's indexEcxOwners).
|
||||
if !ent.metadata().is_ok_and(|m| m.len() > 0) {
|
||||
continue;
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
let key = EcKey { collection, vid };
|
||||
owners.entry(key).or_insert(EcxOwnerInfo {
|
||||
location: loc_idx,
|
||||
@@ -457,18 +450,11 @@ impl Store {
|
||||
///
|
||||
/// Mirrors `Store.CollectEcVolumesMissingIndex` in Go.
|
||||
pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec<EcVolumeMissingIndex> {
|
||||
self.collect_ec_volumes_missing_index_only(None)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_ec_volumes_missing_index_only(
|
||||
&self,
|
||||
only: Option<VolumeId>,
|
||||
) -> Vec<EcVolumeMissingIndex> {
|
||||
let owners = self.index_ecx_owners(only);
|
||||
let owners = self.index_ecx_owners();
|
||||
let mut seen: HashSet<EcKey> = HashSet::new();
|
||||
let mut missing = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
|
||||
for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) {
|
||||
if owners.contains_key(&key) || !seen.insert(key.clone()) {
|
||||
continue;
|
||||
}
|
||||
@@ -488,24 +474,18 @@ impl Store {
|
||||
/// the disks that now have a local index, then fall back to the cross-disk
|
||||
/// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go.
|
||||
pub(crate) fn mount_recovered_ec_shards(&mut self) {
|
||||
self.mount_recovered_ec_shards_only(None)
|
||||
}
|
||||
|
||||
/// `mount_recovered_ec_shards` restricted to one volume's shards when
|
||||
/// `only` names it.
|
||||
pub(crate) fn mount_recovered_ec_shards_only(&mut self, only: Option<VolumeId>) {
|
||||
self.mirror_ec_metadata_to_shard_disks_only(only);
|
||||
self.load_orphan_ec_shards_with_local_index(only);
|
||||
self.reconcile_ec_shards_across_disks_only(only);
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
self.load_orphan_ec_shards_with_local_index();
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
}
|
||||
|
||||
/// Mount on-disk EC shards whose `.ecx` index is now present on the same disk.
|
||||
/// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a
|
||||
/// single-disk store recovers once its index has been fetched from a peer.
|
||||
fn load_orphan_ec_shards_with_local_index(&mut self, only: Option<VolumeId>) {
|
||||
fn load_orphan_ec_shards_with_local_index(&mut self) {
|
||||
let mut work: Vec<(usize, EcKey, Vec<ShardId>)> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx, only) {
|
||||
for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) {
|
||||
if !loc.has_ecx_file_on_disk(&key.collection, key.vid) {
|
||||
continue;
|
||||
}
|
||||
@@ -536,7 +516,6 @@ impl Store {
|
||||
fn collect_orphan_ec_shards(
|
||||
loc: &crate::storage::disk_location::DiskLocation,
|
||||
_loc_idx: usize,
|
||||
only: Option<VolumeId>,
|
||||
) -> HashMap<EcKey, Vec<(String, ShardId)>> {
|
||||
let mut orphans: HashMap<EcKey, Vec<(String, ShardId)>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
@@ -554,12 +533,6 @@ fn collect_orphan_ec_shards(
|
||||
let Some(shard_id) = is_ec_shard_extension(ext) else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
if only.is_some_and(|o| o != vid) {
|
||||
continue;
|
||||
}
|
||||
// Ignore zero-byte shards. Use the DirEntry's metadata so we
|
||||
// don't pay a second stat syscall per file beyond what
|
||||
// read_dir already returned.
|
||||
@@ -567,6 +540,9 @@ fn collect_orphan_ec_shards(
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
// Skip shards that are already registered to an EcVolume.
|
||||
if let Some(ecv) = loc.find_ec_volume(vid)
|
||||
&& ecv.has_shard(shard_id)
|
||||
@@ -679,7 +655,7 @@ mod tests {
|
||||
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(7)), b"").unwrap();
|
||||
write_index_files(&d1, "pics", 7, 10, 4);
|
||||
|
||||
let owners = store.index_ecx_owners(None);
|
||||
let owners = store.index_ecx_owners();
|
||||
let owner = owners
|
||||
.get(&EcKey {
|
||||
collection: "pics".to_string(),
|
||||
@@ -691,7 +667,7 @@ mod tests {
|
||||
|
||||
// A stub with no real index anywhere owns nothing.
|
||||
std::fs::write(ec_local_ecx_path(&d0, "pics", VolumeId(8)), b"").unwrap();
|
||||
assert!(!store.index_ecx_owners(None).contains_key(&EcKey {
|
||||
assert!(!store.index_ecx_owners().contains_key(&EcKey {
|
||||
collection: "pics".to_string(),
|
||||
vid: VolumeId(8),
|
||||
}));
|
||||
@@ -1264,79 +1240,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mount_recovered_ec_shards_only_named_volume() {
|
||||
// Recovering one volume mounts only its shards. Another volume's orphan
|
||||
// shards, such as ones an in-flight copy just renamed in, stay unmounted
|
||||
// even when an index for them is already on disk. Mirrors Go
|
||||
// TestMountRecoveredEcShards_OnlyNamedVolume.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
std::fs::create_dir_all(&dir0).unwrap();
|
||||
std::fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "col";
|
||||
let (recovered_vid, other_vid) = (71u32, 72u32);
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for d in [&dir0, &dir1] {
|
||||
store
|
||||
.add_location(
|
||||
d.to_str().unwrap(),
|
||||
d.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for vid in [recovered_vid, other_vid] {
|
||||
write_shard(dir0.to_str().unwrap(), collection, vid, 0);
|
||||
write_shard(dir1.to_str().unwrap(), collection, vid, 1);
|
||||
}
|
||||
write_index_files(dir0.to_str().unwrap(), collection, other_vid, 10, 4);
|
||||
let missing = store.collect_ec_volumes_missing_index();
|
||||
let m = missing
|
||||
.iter()
|
||||
.find(|m| m.vid == VolumeId(recovered_vid))
|
||||
.expect("recovered volume not reported as missing its index");
|
||||
write_index_files(&m.idx_dir, collection, recovered_vid, 10, 4);
|
||||
|
||||
store.mount_recovered_ec_shards_only(Some(VolumeId(recovered_vid)));
|
||||
|
||||
for (i, loc) in store.locations.iter().enumerate() {
|
||||
let ev = loc
|
||||
.find_ec_volume(VolumeId(recovered_vid))
|
||||
.unwrap_or_else(|| panic!("volume {} not mounted on disk{}", recovered_vid, i));
|
||||
assert!(
|
||||
ev.has_shard(i as u8),
|
||||
"shard {}.{} not mounted",
|
||||
recovered_vid,
|
||||
i
|
||||
);
|
||||
assert!(
|
||||
loc.find_ec_volume(VolumeId(other_vid)).is_none(),
|
||||
"volume {} mounted on disk{} by the recovery of volume {}",
|
||||
other_vid,
|
||||
i,
|
||||
recovered_vid,
|
||||
);
|
||||
}
|
||||
let mirrored = format!(
|
||||
"{}/{}_{}.ecx",
|
||||
dir1.to_str().unwrap(),
|
||||
collection,
|
||||
other_vid
|
||||
);
|
||||
assert!(
|
||||
!std::path::Path::new(&mirrored).exists(),
|
||||
"volume {} index mirrored to disk1 by the recovery of volume {}",
|
||||
other_vid,
|
||||
recovered_vid,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_missing_index_excludes_cross_disk_orphan() {
|
||||
// A .ecx merely on a sibling disk is the same-server case the reconcile
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -188,7 +188,7 @@ mod tests {
|
||||
for id in 1..=3 {
|
||||
v.write_needle(&mut needle(id), true, false).unwrap();
|
||||
}
|
||||
v.delete_needle(&mut needle(2), false).unwrap();
|
||||
v.delete_needle(&mut needle(2)).unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
let (want_count, want_deleted) = (v.file_count(), v.deleted_count());
|
||||
drop(v);
|
||||
|
||||
@@ -382,13 +382,10 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 6);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
@@ -420,13 +417,10 @@ pub(crate) mod tests {
|
||||
|
||||
write_test_volume(dir, 8);
|
||||
let v = open_volume(dir);
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
drop(v);
|
||||
|
||||
|
||||
@@ -505,13 +505,10 @@ mod tests {
|
||||
f.written.insert(*id, n.data.clone());
|
||||
}
|
||||
Op::Del(id) => {
|
||||
v.delete_needle(
|
||||
&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
},
|
||||
false,
|
||||
)
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(*id),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
Op::Empty(id) => {
|
||||
|
||||
@@ -110,7 +110,6 @@ fn build_test_state(
|
||||
vacuum_commit_slots: None,
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
ec_lifecycle_locks: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
volume_copies_in_flight: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
@@ -1436,120 +1435,3 @@ async fn write_queue_upload_returns_same_etag_as_direct_write() {
|
||||
}
|
||||
assert_eq!(etags[1], etags[0], "unchanged upload ETag differs");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// A Range over a compressed needle is answered from the decoded bytes
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn range_with_accept_gzip_on_compressed_needle_serves_decoded_bytes() {
|
||||
let (state, _tmp) = test_state();
|
||||
let uri = "/1/03637037d6/notes.txt";
|
||||
let plain: Vec<u8> = (0..400)
|
||||
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
|
||||
.collect();
|
||||
|
||||
let app = build_admin_router(state.clone());
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.body(Body::from(plain.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
|
||||
let read = |range: Option<&str>| {
|
||||
let mut request = Request::builder()
|
||||
.uri(uri)
|
||||
.header("Accept-Encoding", "gzip");
|
||||
if let Some(range) = range {
|
||||
request = request.header("Range", range);
|
||||
}
|
||||
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
|
||||
};
|
||||
|
||||
// The whole read passes the stored gzip through, so its length is the compressed size.
|
||||
let response = read(None).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()["Content-Encoding"], "gzip");
|
||||
let compressed_len = body_bytes(response).await.len();
|
||||
assert!(compressed_len < plain.len() / 4);
|
||||
|
||||
for (start, end) in [(10, 69), (compressed_len + 10, compressed_len + 69)] {
|
||||
let response = read(Some(&format!("bytes={start}-{end}"))).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT);
|
||||
assert!(response.headers().get("Content-Encoding").is_none());
|
||||
assert_eq!(
|
||||
response.headers()["Content-Range"],
|
||||
format!("bytes {start}-{end}/{}", plain.len()).as_str()
|
||||
);
|
||||
assert_eq!(body_bytes(response).await, &plain[start..=end]);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// HEAD of a compressed needle reports the length the matching GET would send
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_on_compressed_needle_reports_served_length() {
|
||||
let (state, _tmp) = test_state();
|
||||
let uri = "/1/04637037d6/notes.txt";
|
||||
let plain: Vec<u8> = (0..400)
|
||||
.flat_map(|i| format!("line {i:06}: the quick brown fox\n").into_bytes())
|
||||
.collect();
|
||||
|
||||
let app = build_admin_router(state.clone());
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(uri)
|
||||
.body(Body::from(plain.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
|
||||
let send = |method: &str, accept_gzip: bool, range: Option<&str>| {
|
||||
let mut request = Request::builder().method(method).uri(uri);
|
||||
if accept_gzip {
|
||||
request = request.header("Accept-Encoding", "gzip");
|
||||
}
|
||||
if let Some(range) = range {
|
||||
request = request.header("Range", range);
|
||||
}
|
||||
build_admin_router(state.clone()).oneshot(request.body(Body::empty()).unwrap())
|
||||
};
|
||||
|
||||
let response = send("GET", true, None).await.unwrap();
|
||||
let compressed_len = body_bytes(response).await.len();
|
||||
assert!(compressed_len < plain.len() / 4);
|
||||
|
||||
for (accept_gzip, range, encoding, length) in [
|
||||
(false, None, None, plain.len()),
|
||||
(true, None, Some("gzip"), compressed_len),
|
||||
(true, Some("bytes=0-9"), None, plain.len()),
|
||||
] {
|
||||
let response = send("HEAD", accept_gzip, range).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("Content-Encoding")
|
||||
.map(|v| v.to_str().unwrap()),
|
||||
encoding,
|
||||
"accept_gzip={accept_gzip} range={range:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers()["Content-Length"],
|
||||
length.to_string().as_str(),
|
||||
"accept_gzip={accept_gzip} range={range:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Generated
-4
@@ -7157,7 +7157,6 @@ dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -7171,10 +7170,8 @@ dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"http 1.5.0",
|
||||
"hyper-util",
|
||||
"jsonb",
|
||||
"jsonwebtoken",
|
||||
"lance",
|
||||
"lance-arrow",
|
||||
"lance-datafusion",
|
||||
"lance-index",
|
||||
"lance-linalg",
|
||||
@@ -7206,7 +7203,6 @@ dependencies = [
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"x509-parser",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
@@ -3,7 +3,7 @@ use tokio::sync::mpsc;
|
||||
|
||||
use crate::pb::{
|
||||
ActivityEvent, DetectionComplete, DetectionProposals, JobCompleted, JobProgressUpdate,
|
||||
JobState, WorkerObservations, WorkerToAdminMessage, worker_to_admin_message::Body,
|
||||
WorkerObservations, WorkerToAdminMessage, worker_to_admin_message::Body,
|
||||
};
|
||||
|
||||
/// Replies to one detection request.
|
||||
@@ -65,12 +65,7 @@ impl DetectionSender for StreamSender {
|
||||
}
|
||||
|
||||
impl ExecutionSender for StreamSender {
|
||||
fn send_progress(&self, mut progress: JobProgressUpdate) -> Result<()> {
|
||||
// the Go handlers always report a state; older admins read a missing
|
||||
// one as an ended job and drop its completion
|
||||
if progress.state == JobState::Unspecified as i32 {
|
||||
progress.state = JobState::Running as i32;
|
||||
}
|
||||
fn send_progress(&self, progress: JobProgressUpdate) -> Result<()> {
|
||||
self.send(Body::JobProgressUpdate(progress))
|
||||
}
|
||||
|
||||
@@ -205,36 +200,4 @@ mod tests {
|
||||
.expect("send");
|
||||
assert!(metered.reported_failure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_without_a_state_is_sent_as_running() {
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
let stream = StreamSender::new("worker-1".to_string(), tx);
|
||||
let metered = MeteredSender::new(&stream);
|
||||
|
||||
metered
|
||||
.send_progress(JobProgressUpdate {
|
||||
progress_percent: 10.0,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("send");
|
||||
metered
|
||||
.send_progress(JobProgressUpdate {
|
||||
state: JobState::Failed as i32,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("send");
|
||||
|
||||
let mut states = Vec::new();
|
||||
while let Ok(message) = rx.try_recv() {
|
||||
let Some(Body::JobProgressUpdate(progress)) = message.body else {
|
||||
panic!("expected a progress update");
|
||||
};
|
||||
states.push(progress.state);
|
||||
}
|
||||
assert_eq!(
|
||||
states,
|
||||
vec![JobState::Running as i32, JobState::Failed as i32]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -35,16 +35,10 @@ lance-table = "10"
|
||||
# For the execution options the sort needs; the scanner's own stream helper
|
||||
# takes the default, which has spilling off.
|
||||
lance-datafusion = "10"
|
||||
arrow-array = "58"
|
||||
arrow-schema = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-select = "58"
|
||||
# Lance's stored form of the metadata JSON, read when deriving the
|
||||
# indexed-key columns.
|
||||
jsonb = { version = "0.5", default-features = false }
|
||||
chrono = "0.4"
|
||||
futures = "0.3"
|
||||
uuid = "1"
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
@@ -94,10 +88,9 @@ protoc-bin-vendored = "3"
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tonic.workspace = true
|
||||
lance-arrow = "10"
|
||||
arrow-array = "58"
|
||||
arrow-schema = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-select = "58"
|
||||
lance-linalg = "10"
|
||||
tempfile = "3"
|
||||
|
||||
|
||||
@@ -18,8 +18,7 @@ use lance::io::{
|
||||
use lance::session::Session;
|
||||
|
||||
use crate::catalog::{NamespaceClient, TableDescription};
|
||||
use crate::direct_read::{DirectReads, Wrappers};
|
||||
use crate::drop_guard::DropGuard;
|
||||
use crate::direct_read::DirectReads;
|
||||
|
||||
/// A table the worker is about to work on.
|
||||
pub struct OpenTable {
|
||||
@@ -76,7 +75,7 @@ pub fn unsigned_options() -> HashMap<String, String> {
|
||||
pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) -> Result<OpenTable> {
|
||||
let description = client.describe_table(id).await?;
|
||||
let wrapper = storage.wrapper(&description.location).await?;
|
||||
let dataset = open_guarded(&description, wrapper).await?;
|
||||
let dataset = open_at(&description, wrapper, None).await?;
|
||||
Ok(OpenTable {
|
||||
id: id.to_vec(),
|
||||
location: description.location,
|
||||
@@ -84,21 +83,6 @@ pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) ->
|
||||
})
|
||||
}
|
||||
|
||||
/// [`open_at`] for a job that writes the table: its writes fail once the
|
||||
/// table is dropped under it (see [`DropGuard`]).
|
||||
pub async fn open_guarded(
|
||||
description: &TableDescription,
|
||||
wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||
) -> Result<Dataset> {
|
||||
let guard = DropGuard::new();
|
||||
let mut wrappers: Vec<Arc<dyn WrappingObjectStore>> = wrapper.into_iter().collect();
|
||||
wrappers.push(guard.clone());
|
||||
let dataset = open_at(description, Some(Arc::new(Wrappers(wrappers))), None).await?;
|
||||
let opened = dataset.manifest_location();
|
||||
guard.arm(opened.path.clone(), opened.e_tag.clone());
|
||||
Ok(dataset)
|
||||
}
|
||||
|
||||
/// Open a table the caller has already resolved, with the store wrapped, in
|
||||
/// `session` when given, otherwise in a session of its own.
|
||||
pub async fn open_at(
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
//! Keeps a job from writing into a table dropped under it.
|
||||
//!
|
||||
//! DeleteIndex purges a table's directory, and an index created again under
|
||||
//! the same name reuses it. A job that opened the table before the purge
|
||||
//! would otherwise go on writing there: its data files would land in the
|
||||
//! new table's directory, and its commit would put a manifest of the old
|
||||
//! table there, so the new table could not be created, or would read it.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::BoxStream;
|
||||
use lance::io::WrappingObjectStore;
|
||||
use object_store::path::Path;
|
||||
use object_store::{
|
||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
|
||||
UploadPart,
|
||||
};
|
||||
|
||||
/// Fails every write once the manifest it watches is gone or is another
|
||||
/// object (a table created again reuses version paths): the one of the
|
||||
/// version the job opened, then the newest the job committed. Armed after the
|
||||
/// open; until then it lets everything through. When the job itself deletes
|
||||
/// the watched manifest, removing old versions as a purge does, it watches
|
||||
/// the newest manifest left instead.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DropGuard {
|
||||
watched: Arc<Mutex<Watched>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct Watched {
|
||||
manifest: Option<Path>,
|
||||
/// The manifest's e-tag when known, telling it from a replacement.
|
||||
e_tag: Option<String>,
|
||||
/// The job deleted the watched manifest itself.
|
||||
deleted: bool,
|
||||
}
|
||||
|
||||
impl DropGuard {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self::default())
|
||||
}
|
||||
|
||||
/// Watch `manifest`, the opened version's, with its e-tag when known.
|
||||
pub fn arm(&self, manifest: Path, e_tag: Option<String>) {
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
watched.manifest = Some(manifest);
|
||||
watched.e_tag = e_tag;
|
||||
}
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for DropGuard {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(Guarded {
|
||||
inner: original,
|
||||
watched: self.watched.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Guarded {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
watched: Arc<Mutex<Watched>>,
|
||||
}
|
||||
|
||||
/// What a check found: the table is there, or it was dropped, or created
|
||||
/// again, with the error to fail the write with.
|
||||
enum Found {
|
||||
Table,
|
||||
Dropped(object_store::Error),
|
||||
Replaced(object_store::Error),
|
||||
}
|
||||
|
||||
impl Guarded {
|
||||
/// Whether the watched manifest is still the table's. An error is a
|
||||
/// failed probe, which says nothing about the table.
|
||||
async fn find(&self, location: &Path) -> Result<Found> {
|
||||
let watched = self.watched.lock().unwrap().clone();
|
||||
let Some(mut manifest) = watched.manifest else {
|
||||
return Ok(Found::Table);
|
||||
};
|
||||
let mut e_tag = watched.e_tag;
|
||||
if watched.deleted {
|
||||
if let Some((newest, tag)) = self.newest_manifest(&manifest).await? {
|
||||
manifest = newest;
|
||||
e_tag = tag;
|
||||
}
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
watched.manifest = Some(manifest.clone());
|
||||
watched.e_tag = e_tag.clone();
|
||||
watched.deleted = false;
|
||||
}
|
||||
let error = |why: &str| {
|
||||
object_store::Error::Generic {
|
||||
store: "DropGuard",
|
||||
source: format!(
|
||||
"the table was dropped while this job wrote it: {manifest} {why}, so {location} is not written"
|
||||
)
|
||||
.into(),
|
||||
}
|
||||
};
|
||||
match self.inner.head(&manifest).await {
|
||||
Ok(meta) => match (&e_tag, &meta.e_tag) {
|
||||
(Some(want), Some(have)) if want != have => {
|
||||
Ok(Found::Replaced(error("is another table's")))
|
||||
}
|
||||
_ => Ok(Found::Table),
|
||||
},
|
||||
// Gone with manifests beside it: a table created again, at an
|
||||
// earlier version than the one watched.
|
||||
Err(object_store::Error::NotFound { .. }) => {
|
||||
match self.newest_manifest(&manifest).await? {
|
||||
Some(_) => Ok(Found::Replaced(error("is gone and another table is there"))),
|
||||
None => Ok(Found::Dropped(error("is gone"))),
|
||||
}
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn check(&self, location: &Path) -> Result<()> {
|
||||
match self.find(location).await? {
|
||||
Found::Table => Ok(()),
|
||||
Found::Dropped(error) | Found::Replaced(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_manifest(location: &Path) -> bool {
|
||||
location.extension() == Some("manifest")
|
||||
}
|
||||
|
||||
impl Guarded {
|
||||
/// The most recently written manifest beside `manifest`, with its e-tag.
|
||||
async fn newest_manifest(&self, manifest: &Path) -> Result<Option<(Path, Option<String>)>> {
|
||||
let dir: Path = manifest
|
||||
.parts()
|
||||
.take(manifest.parts().count() - 1)
|
||||
.collect();
|
||||
let listed = self.inner.list_with_delimiter(Some(&dir)).await?;
|
||||
Ok(listed
|
||||
.objects
|
||||
.into_iter()
|
||||
.filter(|o| is_manifest(&o.location))
|
||||
.max_by_key(|o| o.last_modified)
|
||||
.map(|o| (o.location, o.e_tag)))
|
||||
}
|
||||
|
||||
fn committed(&self, location: &Path, e_tag: Option<String>) {
|
||||
if is_manifest(location) {
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
if watched.manifest.is_some() {
|
||||
watched.manifest = Some(location.clone());
|
||||
watched.e_tag = e_tag;
|
||||
watched.deleted = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A multipart upload that checks the table again before it completes.
|
||||
#[derive(Debug)]
|
||||
struct GuardedUpload {
|
||||
inner: Box<dyn MultipartUpload>,
|
||||
guard: Guarded,
|
||||
location: Path,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MultipartUpload for GuardedUpload {
|
||||
fn put_part(&mut self, data: PutPayload) -> UploadPart {
|
||||
self.inner.put_part(data)
|
||||
}
|
||||
|
||||
async fn complete(&mut self) -> Result<PutResult> {
|
||||
// A failed probe aborts too: the parts would outlive the upload.
|
||||
let error = match self.guard.find(&self.location).await {
|
||||
Ok(Found::Table) => return self.inner.complete().await,
|
||||
Ok(Found::Dropped(error) | Found::Replaced(error)) | Err(error) => error,
|
||||
};
|
||||
let _ = self.inner.abort().await;
|
||||
Err(error)
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> Result<()> {
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Guarded {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "DropGuard({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for Guarded {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
self.check(location).await?;
|
||||
let put = self.inner.put_opts(location, payload, opts).await?;
|
||||
// A drop can land between the check and the write. A manifest is
|
||||
// what makes the table readable again, so look once more and take
|
||||
// it back if the table is gone; a failed probe leaves it. A table
|
||||
// created again since may have written its own manifest there:
|
||||
// only a manifest known to be this write's is taken back.
|
||||
if is_manifest(location) {
|
||||
let (error, replaced) = match self.find(location).await {
|
||||
Ok(Found::Dropped(error)) => (error, false),
|
||||
Ok(Found::Replaced(error)) => (error, true),
|
||||
_ => {
|
||||
self.committed(location, put.e_tag.clone());
|
||||
return Ok(put);
|
||||
}
|
||||
};
|
||||
let ours = match (&put.e_tag, self.inner.head(location).await) {
|
||||
(Some(wrote), Ok(meta)) => meta.e_tag.as_ref() == Some(wrote),
|
||||
(_, Err(object_store::Error::NotFound { .. })) => false,
|
||||
_ => !replaced,
|
||||
};
|
||||
if ours {
|
||||
let _ = self.inner.delete(location).await;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
self.committed(location, put.e_tag.clone());
|
||||
Ok(put)
|
||||
}
|
||||
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
self.check(location).await?;
|
||||
let inner = self.inner.put_multipart_opts(location, opts).await?;
|
||||
Ok(Box::new(GuardedUpload {
|
||||
inner,
|
||||
guard: self.clone(),
|
||||
location: location.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
// A cleanup picks its paths before it deletes them; a table created
|
||||
// again meanwhile reuses those paths, so look once before deleting.
|
||||
// In a dropped table there is nothing left to spare, and the job's
|
||||
// own failed commit is cleaned up.
|
||||
let watched = self.watched.clone();
|
||||
let locations = locations.inspect(move |location| {
|
||||
if let Ok(location) = location {
|
||||
let mut watched = watched.lock().unwrap();
|
||||
if watched.manifest.as_ref() == Some(location) {
|
||||
watched.deleted = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
let guard = self.clone();
|
||||
let pending = Arc::new(Mutex::new(Some(locations.boxed())));
|
||||
let checked =
|
||||
futures::stream::once(async move { guard.find(&Path::from("cleanup")).await })
|
||||
.flat_map(move |found| {
|
||||
let locations = pending.lock().unwrap().take();
|
||||
match (found, locations) {
|
||||
(Ok(Found::Table | Found::Dropped(_)), Some(locations)) => locations,
|
||||
(Ok(Found::Replaced(error)), _) | (Err(error), _) => {
|
||||
futures::stream::once(async move { Err(error) }).boxed()
|
||||
}
|
||||
(Ok(_), None) => futures::stream::empty().boxed(),
|
||||
}
|
||||
})
|
||||
.boxed();
|
||||
self.inner.delete_stream(checked)
|
||||
}
|
||||
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.check(to).await?;
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.check(to).await?;
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
@@ -1,701 +0,0 @@
|
||||
//! A table dropped while a worker job runs on it: the job must fail without
|
||||
//! writing into the dropped path, and a table created again there starts
|
||||
//! empty.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::BoxStream;
|
||||
use lance::Dataset;
|
||||
use lance::io::WrappingObjectStore;
|
||||
use object_store::path::Path;
|
||||
use object_store::{
|
||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
|
||||
};
|
||||
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
use crate::metadata_index::{self, State};
|
||||
|
||||
/// Removes the table directory just before the first write through it, the
|
||||
/// way DeleteIndex purging the table between a job's open and its writes
|
||||
/// would.
|
||||
#[derive(Debug)]
|
||||
struct DropOnFirstWrite {
|
||||
dir: PathBuf,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for DropOnFirstWrite {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(Dropping {
|
||||
inner: original,
|
||||
dir: self.dir.clone(),
|
||||
fired: self.fired.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Dropping {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
dir: PathBuf,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Dropping {
|
||||
fn drop_once(&self) {
|
||||
if !self.fired.swap(true, Ordering::SeqCst) {
|
||||
std::fs::remove_dir_all(&self.dir).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Dropping {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "Dropping({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for Dropping {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
self.drop_once();
|
||||
self.inner.put_opts(location, payload, opts).await
|
||||
}
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
self.drop_once();
|
||||
self.inner.put_multipart_opts(location, opts).await
|
||||
}
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
self.inner.delete_stream(locations)
|
||||
}
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens the table the way a job does; `during` drops it at the job's first
|
||||
/// write, otherwise it is dropped right after the open.
|
||||
async fn open_then_drop(dir: &tempfile::TempDir, during: bool) -> Dataset {
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let dropping: Option<Arc<dyn WrappingObjectStore>> = during.then(|| {
|
||||
Arc::new(DropOnFirstWrite {
|
||||
dir: dir.path().to_path_buf(),
|
||||
fired: Arc::new(AtomicBool::new(false)),
|
||||
}) as Arc<dyn WrappingObjectStore>
|
||||
});
|
||||
let dataset = crate::dataset::open_guarded(&description, dropping)
|
||||
.await
|
||||
.unwrap();
|
||||
if !during {
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
}
|
||||
dataset
|
||||
}
|
||||
|
||||
fn files(dir: &std::path::Path) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if let Ok(entries) = std::fs::read_dir(dir) {
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
out.extend(files(&path));
|
||||
} else {
|
||||
out.push(path.display().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Runs `job` on a table dropped right after the job opens it, or at the
|
||||
/// job's first write. The job must fail, leave no manifest (nor, dropped
|
||||
/// before it wrote, any file) in the dropped path, and a table created
|
||||
/// there again must start empty.
|
||||
async fn dropped_during<F, Fut>(
|
||||
label: &str,
|
||||
during: bool,
|
||||
prepare: impl AsyncFnOnce(&mut Dataset),
|
||||
job: F,
|
||||
) where
|
||||
F: FnOnce(Dataset) -> Fut,
|
||||
Fut: std::future::Future<Output = anyhow::Result<()>>,
|
||||
{
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
prepare(&mut dataset).await;
|
||||
let result = job(open_then_drop(&dir, during).await).await;
|
||||
let left = files(dir.path());
|
||||
let error = result.expect_err(&format!(
|
||||
"{label}: a job on a dropped table succeeded, leaving {left:?}"
|
||||
));
|
||||
eprintln!(
|
||||
"{label} (dropped {}): {error:#}",
|
||||
if during {
|
||||
"at its first write"
|
||||
} else {
|
||||
"after the open"
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
!left.iter().any(|f| f.ends_with(".manifest")),
|
||||
"{label}: a manifest was written into the dropped table: {left:?}"
|
||||
);
|
||||
if !during {
|
||||
assert!(
|
||||
left.is_empty(),
|
||||
"{label}: files were written into the dropped table: {left:?}"
|
||||
);
|
||||
}
|
||||
let recreated = Dataset::write(
|
||||
arrow_array::RecordBatchIterator::new(
|
||||
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
|
||||
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
|
||||
),
|
||||
dir.path().to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("{label}: the table could not be created again: {e}"));
|
||||
assert_eq!(recreated.version().version, 1, "{label}");
|
||||
assert_eq!(recreated.count_rows(None).await.unwrap(), 0, "{label}");
|
||||
}
|
||||
|
||||
/// Every job, with the table dropped after its open and at its first write.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_job_on_a_dropped_table_fails_and_leaves_it_dropped() {
|
||||
for during in [false, true] {
|
||||
dropped_during(
|
||||
"vector index build",
|
||||
during,
|
||||
async |_| {},
|
||||
async |mut d| {
|
||||
crate::vector_index::build(
|
||||
&mut d,
|
||||
"vector",
|
||||
crate::vector_index::INDEX_NAME,
|
||||
4,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key add",
|
||||
during,
|
||||
async |d| metadata_index::request(d, "author").await.unwrap(),
|
||||
async |mut d| {
|
||||
let key = metadata_index::add_column(&mut d, "author").await?;
|
||||
metadata_index::finish_after(&mut d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.map(|_| ())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key maintenance",
|
||||
during,
|
||||
async |d| {
|
||||
let key = metadata_index::add_column(d, "author").await.unwrap();
|
||||
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
*d = upsert(d.clone(), 4_000..4_500).await;
|
||||
},
|
||||
async |mut d| {
|
||||
let key = metadata_index::keys_of(&d).remove(0);
|
||||
metadata_index::maintain(&mut d, &key, 1_000, 0)
|
||||
.await
|
||||
.map(|_| ())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key remove",
|
||||
during,
|
||||
async |d| {
|
||||
let key = metadata_index::add_column(d, "author").await.unwrap();
|
||||
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::set_state(d, "_mk_0", State::Off)
|
||||
.await
|
||||
.unwrap();
|
||||
},
|
||||
async |mut d| metadata_index::remove_column(&mut d, "_mk_0").await,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A job that removes old versions itself, as a purge does, removes the
|
||||
/// manifest it opened: that is no drop, and its later writes go through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_purge_through_the_guard_completes() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let dataset = table(&dir, 4).await;
|
||||
let mut dataset = upsert(dataset, 0..100).await;
|
||||
dataset.delete("key = 'k1' OR key = 'k2001'").await.unwrap();
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let mut guarded = crate::dataset::open_guarded(&description, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// The job commits, then removes every older version, the one it opened
|
||||
// with them.
|
||||
guarded
|
||||
.update_config([("seaweedfs.test", "1")])
|
||||
.await
|
||||
.unwrap();
|
||||
let removed = lance::dataset::cleanup::cleanup_old_versions(
|
||||
&guarded,
|
||||
lance::dataset::cleanup::CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now() + chrono::Duration::hours(1)),
|
||||
before_version: Some(guarded.version().version),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(removed.old_versions > 0);
|
||||
guarded
|
||||
.update_config([("seaweedfs.test", "2")])
|
||||
.await
|
||||
.unwrap();
|
||||
crate::jobs::purge::purge(&mut guarded, "t1").await.unwrap();
|
||||
assert_eq!(guarded.count_rows(None).await.unwrap(), 3_998);
|
||||
}
|
||||
|
||||
/// The table is dropped and created again up to the version the job opened,
|
||||
/// so the manifest path the job watches exists again, as another object:
|
||||
/// the job must still fail and leave the new table as it was.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_job_on_a_table_created_again_fails() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
let key = metadata_index::add_column(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::finish_after(&mut dataset, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::set_state(&mut dataset, "_mk_0", State::Off)
|
||||
.await
|
||||
.unwrap();
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let mut job = crate::dataset::open_guarded(&description, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let opened = job.version().version;
|
||||
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let mut recreated = table(&dir, 1).await;
|
||||
while recreated.version().version < opened {
|
||||
let version = recreated.version().version.to_string();
|
||||
recreated
|
||||
.update_config([("seaweedfs.test", version.as_str())])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(recreated.version().version, opened);
|
||||
|
||||
let error = metadata_index::remove_column(&mut job, "_mk_0")
|
||||
.await
|
||||
.expect_err("a job on a table created again succeeded");
|
||||
assert!(format!("{error:#}").contains("DropGuard"), "{error:#}");
|
||||
recreated.checkout_latest().await.unwrap();
|
||||
assert_eq!(recreated.version().version, opened);
|
||||
assert_eq!(recreated.count_rows(None).await.unwrap(), 1_000);
|
||||
}
|
||||
|
||||
fn guarded_description(dir: &tempfile::TempDir) -> crate::catalog::TableDescription {
|
||||
crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The table's directory in the store's own paths.
|
||||
fn table_root(dataset: &Dataset) -> Path {
|
||||
let manifest = &dataset.manifest_location().path;
|
||||
manifest
|
||||
.parts()
|
||||
.take(manifest.parts().count() - 2)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A multipart upload started before the drop does not complete after it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_multipart_upload_does_not_complete_into_a_dropped_table() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like::default());
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
let path = table_root(&job).child("data").child("late.lance");
|
||||
let mut upload = store.put_multipart(&path).await.unwrap();
|
||||
upload
|
||||
.put_part(PutPayload::from(vec![0u8; 16]))
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let completed = upload.complete().await;
|
||||
assert!(
|
||||
completed.is_err(),
|
||||
"an upload completed into a dropped table"
|
||||
);
|
||||
assert!(files(dir.path()).is_empty(), "{:?}", files(dir.path()));
|
||||
}
|
||||
|
||||
/// An upload whose last probe fails is aborted, not left with its parts.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_probe_aborts_an_upload() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like = S3Like::default();
|
||||
let (armed, aborted) = (s3_like.armed.clone(), s3_like.aborted.clone());
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(s3_like);
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
let path = table_root(&job).child("data").child("late.lance");
|
||||
let mut upload = store.put_multipart(&path).await.unwrap();
|
||||
upload
|
||||
.put_part(PutPayload::from(vec![0u8; 16]))
|
||||
.await
|
||||
.unwrap();
|
||||
armed.store(true, Ordering::SeqCst);
|
||||
assert!(upload.complete().await.is_err());
|
||||
assert!(aborted.load(Ordering::SeqCst), "the upload was left open");
|
||||
}
|
||||
|
||||
/// A table created again, through the version a job just wrote, between
|
||||
/// that write and its check keeps its own manifest at that path.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_rollback_keeps_the_manifest_of_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(job.version().version, 1);
|
||||
assert!(
|
||||
job.update_config([("seaweedfs.test", "job")])
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(again.version().version, 2);
|
||||
assert_eq!(
|
||||
again.config().get("seaweedfs.test").map(String::as_str),
|
||||
Some("again")
|
||||
);
|
||||
}
|
||||
|
||||
/// A table created again whose manifest at the job's path cannot be told
|
||||
/// from the job's, because the HEAD fails, keeps it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_rollback_probe_keeps_the_manifest_of_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
|
||||
fail_head_after_manifest: true,
|
||||
heads_before_failing: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
job.update_config([("seaweedfs.test", "job")])
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(again.version().version, 2);
|
||||
assert_eq!(
|
||||
again.config().get("seaweedfs.test").map(String::as_str),
|
||||
Some("again")
|
||||
);
|
||||
}
|
||||
|
||||
/// A cleanup that picked its paths before the drop does not delete the files
|
||||
/// of the table created again there.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_cleanup_does_not_delete_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 1).await;
|
||||
dataset
|
||||
.update_config([("seaweedfs.test", "1")])
|
||||
.await
|
||||
.unwrap();
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
// The old version-1 manifest, as a cleanup would have picked it.
|
||||
let mut first = dataset.checkout_version(1).await.unwrap();
|
||||
let old = first.manifest_location().path.clone();
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let recreated = table(&dir, 1).await;
|
||||
assert_eq!(recreated.manifest_location().path, old);
|
||||
let deleted: Vec<_> = store
|
||||
.delete_stream(futures::stream::iter(vec![Ok(old.clone())]).boxed())
|
||||
.collect()
|
||||
.await;
|
||||
assert!(deleted.iter().any(|r| r.is_err()), "{deleted:?}");
|
||||
first = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(first.version().version, 1);
|
||||
}
|
||||
|
||||
/// A probe that fails after a manifest was written says nothing about the
|
||||
/// table: the manifest stays and the commit succeeds.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_probe_does_not_take_back_a_commit() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let flaky: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
fail_head_after_manifest: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(flaky))
|
||||
.await
|
||||
.unwrap();
|
||||
let before = job.version().version;
|
||||
job.update_config([("seaweedfs.test", "1")]).await.unwrap();
|
||||
let reopened = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(reopened.version().version, before + 1);
|
||||
}
|
||||
|
||||
/// Behaves as an object store, not a file system: a multipart upload holds
|
||||
/// its parts until it completes, and records an abort. Optionally fails one
|
||||
/// HEAD after a manifest is written, as a timeout would, or creates the table
|
||||
/// again, through two versions, right after the first manifest is written.
|
||||
#[derive(Debug, Default)]
|
||||
struct S3Like {
|
||||
fail_head_after_manifest: bool,
|
||||
/// HEADs let through after a manifest is written before one fails.
|
||||
heads_before_failing: usize,
|
||||
/// The next HEAD fails.
|
||||
armed: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for S3Like {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(S3LikeStore {
|
||||
inner: original,
|
||||
fail_head_after_manifest: self.fail_head_after_manifest,
|
||||
heads_before_failing: self.heads_before_failing,
|
||||
heads_left: Arc::new(AtomicUsize::new(0)),
|
||||
armed: self.armed.clone(),
|
||||
aborted: self.aborted.clone(),
|
||||
recreate: self.recreate.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct S3LikeStore {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
fail_head_after_manifest: bool,
|
||||
heads_before_failing: usize,
|
||||
heads_left: Arc<AtomicUsize>,
|
||||
armed: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
|
||||
}
|
||||
|
||||
impl fmt::Display for S3LikeStore {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "S3Like({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for S3LikeStore {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
let put = self.inner.put_opts(location, payload, opts).await?;
|
||||
if location.extension() == Some("manifest") {
|
||||
if self.fail_head_after_manifest {
|
||||
self.heads_left
|
||||
.store(self.heads_before_failing, Ordering::SeqCst);
|
||||
self.armed.store(true, Ordering::SeqCst);
|
||||
}
|
||||
if let Some((dir, fired)) = &self.recreate
|
||||
&& !fired.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
let mut again = create_empty(dir).await;
|
||||
again
|
||||
.update_config([("seaweedfs.test", "again")])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
Ok(put)
|
||||
}
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
_opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
Ok(Box::new(HeldParts {
|
||||
inner: self.inner.clone(),
|
||||
location: location.clone(),
|
||||
parts: Vec::new(),
|
||||
aborted: self.aborted.clone(),
|
||||
}))
|
||||
}
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
if options.head
|
||||
&& self.armed.load(Ordering::SeqCst)
|
||||
&& self
|
||||
.heads_left
|
||||
.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
|
||||
.is_err()
|
||||
&& self.armed.swap(false, Ordering::SeqCst)
|
||||
{
|
||||
return Err(object_store::Error::Generic {
|
||||
store: "S3Like",
|
||||
source: "timed out".into(),
|
||||
});
|
||||
}
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
self.inner.delete_stream(locations)
|
||||
}
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Parts held outside the table until the upload completes, as S3 does.
|
||||
#[derive(Debug)]
|
||||
struct HeldParts {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
location: Path,
|
||||
parts: Vec<Bytes>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MultipartUpload for HeldParts {
|
||||
fn put_part(&mut self, data: PutPayload) -> object_store::UploadPart {
|
||||
self.parts.extend(data);
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
async fn complete(&mut self) -> Result<PutResult> {
|
||||
let payload: PutPayload = self.parts.drain(..).collect();
|
||||
self.inner.put(&self.location, payload).await
|
||||
}
|
||||
async fn abort(&mut self) -> Result<()> {
|
||||
self.parts.clear();
|
||||
self.aborted.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An empty table at `dir`.
|
||||
async fn create_empty(dir: &std::path::Path) -> Dataset {
|
||||
Dataset::write(
|
||||
arrow_array::RecordBatchIterator::new(
|
||||
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
|
||||
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
|
||||
),
|
||||
dir.to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
@@ -3,12 +3,11 @@ use std::collections::HashMap;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{CleanupPolicy, cleanup_old_versions};
|
||||
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field};
|
||||
use seaweed_worker_core::pb::{
|
||||
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
|
||||
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest, JobCompleted,
|
||||
JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
RunDetectionRequest,
|
||||
};
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
@@ -16,7 +15,6 @@ use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::purge::{self, Retention};
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
|
||||
pub const JOB_TYPE: &str = "lance_cleanup_versions";
|
||||
@@ -30,9 +28,7 @@ const MAX_MIN_VERSIONS: i64 = 1000;
|
||||
|
||||
/// Lance keeps every version until something removes it. Lance can also do this
|
||||
/// itself through auto-cleanup, so this job is for deployments that would rather
|
||||
/// the cluster owned the policy than each writer. A table may override the
|
||||
/// retention in its own config, and may ask for a purge (see [`purge`]), which
|
||||
/// this job carries out ahead of the retention window.
|
||||
/// the cluster owned the policy than each writer.
|
||||
pub struct CleanupVersionsHandler {
|
||||
namespace_url: String,
|
||||
storage: dataset::Storage,
|
||||
@@ -72,8 +68,9 @@ impl CleanupVersionsHandler {
|
||||
|
||||
/// The oldest version that may be removed while still leaving `min_versions`
|
||||
/// behind, or None when the table has no more than the floor.
|
||||
async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<u64>> {
|
||||
let mut versions: Vec<u64> = dataset
|
||||
async fn version_floor(table: &dataset::OpenTable, min_versions: usize) -> Result<Option<u64>> {
|
||||
let mut versions: Vec<u64> = table
|
||||
.dataset
|
||||
.versions()
|
||||
.await?
|
||||
.iter()
|
||||
@@ -86,33 +83,6 @@ async fn version_floor(dataset: &Dataset, min_versions: usize) -> Result<Option<
|
||||
Ok(Some(versions[versions.len() - min_versions]))
|
||||
}
|
||||
|
||||
/// The cleanup that keeps `min_versions` and every version younger than
|
||||
/// `retain_hours`, or None when the floor leaves nothing to remove. A purge
|
||||
/// cleans up through it too.
|
||||
pub async fn retention_policy(
|
||||
dataset: &Dataset,
|
||||
retain_hours: i64,
|
||||
min_versions: usize,
|
||||
) -> Result<Option<CleanupPolicy>> {
|
||||
let Some(floor) = version_floor(dataset, min_versions).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(CleanupPolicy {
|
||||
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
|
||||
// should_clean() ANDs its clauses, so a version has to be both older
|
||||
// than the window and below the floor to go.
|
||||
before_version: Some(floor),
|
||||
// Files this dataset cannot account for are left alone: they may
|
||||
// belong to a writer that has not committed yet, and deleting them
|
||||
// would corrupt a commit in flight.
|
||||
delete_unverified: false,
|
||||
// A tagged version is pinned on purpose, so refuse rather than
|
||||
// silently dropping what someone named.
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for CleanupVersionsHandler {
|
||||
fn capability(&self) -> JobTypeCapability {
|
||||
@@ -164,12 +134,6 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
defaults.clone(),
|
||||
)),
|
||||
worker_default_values: defaults,
|
||||
// A purge compacts and retrains, so it gets a build's time.
|
||||
admin_runtime_defaults: Some(AdminRuntimeDefaults {
|
||||
job_type_max_runtime_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
|
||||
execution_timeout_seconds: super::vector_index::EXECUTION_TIMEOUT_SECONDS,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -217,23 +181,10 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let config = table.dataset.config();
|
||||
let pending_purge = purge::pending(config);
|
||||
let floor = Retention::for_table(
|
||||
config,
|
||||
Retention {
|
||||
retain_hours: DEFAULT_RETAIN_HOURS,
|
||||
min_versions,
|
||||
},
|
||||
MAX_RETAIN_HOURS,
|
||||
MAX_MIN_VERSIONS,
|
||||
)
|
||||
.min_versions;
|
||||
// Age is decided at execution against the retention window; a table
|
||||
// at or under the floor cannot lose a version whatever its age, so
|
||||
// proposing one would only produce a job with nothing to do. A
|
||||
// requested purge ignores the floor: it keeps only the latest.
|
||||
if pending_purge.is_none() && stats.total_versions <= floor {
|
||||
// proposing one would only produce a job with nothing to do.
|
||||
if stats.total_versions <= min_versions {
|
||||
continue;
|
||||
}
|
||||
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
|
||||
@@ -242,21 +193,12 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
proposal_id: format!("{JOB_TYPE}:{encoded}"),
|
||||
dedupe_key: format!("{JOB_TYPE}:{encoded}"),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
summary: match &pending_purge {
|
||||
Some(_) => format!("Purge {encoded} ({} versions)", stats.total_versions),
|
||||
None => format!("Clean up {encoded} ({} versions)", stats.total_versions),
|
||||
},
|
||||
detail: match &pending_purge {
|
||||
Some(token) => format!(
|
||||
"purge requested at {token}: rewrite fragments with deletions \
|
||||
and keep only the latest version"
|
||||
),
|
||||
None => format!(
|
||||
"{} versions retained, above the {floor} floor; \
|
||||
those outside the retention window can go",
|
||||
stats.total_versions
|
||||
),
|
||||
},
|
||||
summary: format!("Clean up {encoded} ({} versions)", stats.total_versions),
|
||||
detail: format!(
|
||||
"{} versions retained, above the {min_versions} floor; \
|
||||
those outside the retention window can go",
|
||||
stats.total_versions
|
||||
),
|
||||
parameters,
|
||||
..Default::default()
|
||||
});
|
||||
@@ -312,33 +254,14 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
) as usize;
|
||||
|
||||
let client = self.client();
|
||||
let mut table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let before = table.stats().await?;
|
||||
|
||||
if let Some(token) = purge::pending(table.dataset.config()) {
|
||||
return self
|
||||
.execute_purge(request, sender, &job.job_id, &mut table, &token)
|
||||
.await;
|
||||
}
|
||||
let Retention {
|
||||
retain_hours,
|
||||
min_versions,
|
||||
} = Retention::for_table(
|
||||
table.dataset.config(),
|
||||
Retention {
|
||||
retain_hours,
|
||||
min_versions,
|
||||
},
|
||||
MAX_RETAIN_HOURS,
|
||||
MAX_MIN_VERSIONS,
|
||||
);
|
||||
|
||||
// The floor is a promise about how much history survives, so it has to
|
||||
// be applied here and not only when the job was proposed: by the time it
|
||||
// runs, versions may have aged past the retention window, and age alone
|
||||
// would take the table below what the operator asked to keep.
|
||||
let Some(policy) = retention_policy(&table.dataset, retain_hours, min_versions).await?
|
||||
else {
|
||||
let Some(floor) = version_floor(&table, min_versions).await? else {
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
@@ -365,6 +288,20 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
let policy = CleanupPolicy {
|
||||
before_timestamp: Some(Utc::now() - Duration::hours(retain_hours)),
|
||||
// should_clean() ANDs its clauses, so a version has to be both older
|
||||
// than the window and below the floor to go.
|
||||
before_version: Some(floor),
|
||||
// Files this dataset cannot account for are left alone: they may
|
||||
// belong to a writer that has not committed yet, and deleting them
|
||||
// would corrupt a commit in flight.
|
||||
delete_unverified: false,
|
||||
// A tagged version is pinned on purpose, so refuse rather than
|
||||
// silently dropping what someone named.
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
};
|
||||
let stats = cleanup_old_versions(&table.dataset, policy)
|
||||
.await
|
||||
.with_context(|| format!("clean up versions of {}", table.location))?;
|
||||
@@ -406,73 +343,3 @@ impl JobHandler for CleanupVersionsHandler {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl CleanupVersionsHandler {
|
||||
async fn execute_purge(
|
||||
&self,
|
||||
request: &ExecuteJobRequest,
|
||||
sender: &dyn ExecutionSender,
|
||||
job_id: &str,
|
||||
table: &mut dataset::OpenTable,
|
||||
token: &str,
|
||||
) -> Result<()> {
|
||||
let stats = purge::purge(&mut table.dataset, token)
|
||||
.await
|
||||
.with_context(|| format!("purge {}", table.location))?;
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.versions_removed.inc_by(stats.old_versions);
|
||||
counters.bytes_reclaimed.inc_by(stats.bytes_removed);
|
||||
}
|
||||
let mut output: HashMap<String, ConfigValue> = HashMap::new();
|
||||
output.insert(
|
||||
"fragments_rewritten".to_string(),
|
||||
int_value(stats.fragments_rewritten as i64),
|
||||
);
|
||||
output.insert(
|
||||
"old_versions_removed".to_string(),
|
||||
int_value(stats.old_versions as i64),
|
||||
);
|
||||
output.insert(
|
||||
"bytes_removed".to_string(),
|
||||
int_value(stats.bytes_removed as i64),
|
||||
);
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job_id.to_string(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
result: Some(JobResult {
|
||||
output_values: output,
|
||||
summary: if stats.served {
|
||||
format!(
|
||||
"purged: rewrote {} fragments, removed {} versions and {} bytes",
|
||||
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"purge pending: rewrote {} fragments, removed {} versions and {} bytes; \
|
||||
versions from before the rebuild are still retained",
|
||||
stats.fragments_rewritten, stats.old_versions, stats.bytes_removed
|
||||
)
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_purge_gets_a_builds_execution_timeout() {
|
||||
let descriptor = CleanupVersionsHandler::new(String::new()).descriptor();
|
||||
let defaults = descriptor.admin_runtime_defaults.unwrap();
|
||||
assert_eq!(defaults.execution_timeout_seconds, 3600);
|
||||
assert_eq!(defaults.job_type_max_runtime_seconds, 3600);
|
||||
assert!(!defaults.enabled);
|
||||
}
|
||||
}
|
||||
@@ -1,757 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use anyhow::{Result, anyhow, bail};
|
||||
use async_trait::async_trait;
|
||||
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field, string_value};
|
||||
use seaweed_worker_core::pb::{
|
||||
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
|
||||
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
RunDetectionRequest, config_value::Kind,
|
||||
};
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::vector_index::{
|
||||
EXECUTION_TIMEOUT_SECONDS, declare_build_runtime, is_vector_table,
|
||||
};
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
use crate::metadata_index::{self, DEFAULT_MIN_UNINDEXED_ROWS, DEFAULT_SEGMENT_ROWS, State};
|
||||
|
||||
pub const JOB_TYPE: &str = "lance_metadata_index";
|
||||
|
||||
const SEGMENT_ROWS_FLOOR: i64 = 1_000;
|
||||
const SEGMENT_ROWS_CEILING: i64 = 1_000_000_000;
|
||||
const MIN_UNINDEXED_FLOOR: i64 = 0;
|
||||
const MIN_UNINDEXED_CEILING: i64 = 100_000_000;
|
||||
|
||||
/// What a job does to one key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Action {
|
||||
/// Add the requested key's column, or finish a `building` one.
|
||||
Add,
|
||||
/// Drop an `off` key's index and column.
|
||||
Remove,
|
||||
/// Fill NULL rows and fold unindexed fragments of a `ready` key.
|
||||
Maintain,
|
||||
}
|
||||
|
||||
impl Action {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Action::Add => "add",
|
||||
Action::Remove => "remove",
|
||||
Action::Maintain => "maintain",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"add" => Some(Action::Add),
|
||||
"remove" => Some(Action::Remove),
|
||||
"maintain" => Some(Action::Maintain),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The work an index's table asks for: requested and `building` keys to add,
|
||||
/// `off` keys to remove, and `ready` keys whose unindexed rows reached
|
||||
/// `min_unindexed_rows`.
|
||||
pub async fn pending(
|
||||
dataset: &lance::Dataset,
|
||||
min_unindexed_rows: u64,
|
||||
) -> Result<Vec<(String, Action)>> {
|
||||
let mut out: Vec<(String, Action)> = metadata_index::requests(dataset)
|
||||
.into_iter()
|
||||
.map(|key| (key, Action::Add))
|
||||
.collect();
|
||||
for key in metadata_index::keys_of(dataset) {
|
||||
let action = match key.state {
|
||||
Some(State::Building) => Action::Add,
|
||||
Some(State::Off) => Action::Remove,
|
||||
Some(State::Ready) => {
|
||||
let segments = metadata_index::segments(dataset, &key.column).await?;
|
||||
let pending = metadata_index::unindexed(dataset, &segments);
|
||||
let unindexed: u64 = pending.iter().map(|(_, rows)| rows).sum();
|
||||
if unindexed == 0 {
|
||||
continue;
|
||||
}
|
||||
// Rows a write left NULL (one that raced the key's add and
|
||||
// stopped before fixing them) wait for no threshold: queries
|
||||
// through the key miss them until they are derived again.
|
||||
// Nor does the first segment of a key that went ready on an
|
||||
// empty table.
|
||||
if unindexed < min_unindexed_rows
|
||||
&& !segments.is_empty()
|
||||
&& !has_nulls(dataset, &key.column, &pending).await?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Action::Maintain
|
||||
}
|
||||
None => continue,
|
||||
};
|
||||
out.push((key.key, action));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Whether `fragments` hold a row whose `column` is NULL.
|
||||
async fn has_nulls(
|
||||
dataset: &lance::Dataset,
|
||||
column: &str,
|
||||
fragments: &[(u32, u64)],
|
||||
) -> Result<bool> {
|
||||
let ids: Vec<u64> = fragments.iter().map(|(id, _)| *id as u64).collect();
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.filter(&format!("`{}` IS NULL", column.replace('`', "``")))?
|
||||
.with_fragments(
|
||||
dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.filter(|f| ids.contains(&f.id))
|
||||
.cloned()
|
||||
.collect(),
|
||||
);
|
||||
Ok(scanner.count_rows().await? > 0)
|
||||
}
|
||||
|
||||
/// Carries out `action` on `key`, as the table now records it; an action the
|
||||
/// table no longer asks for is skipped. Returns what was done.
|
||||
pub async fn run(
|
||||
dataset: &mut lance::Dataset,
|
||||
key: &str,
|
||||
action: Action,
|
||||
segment_rows: u64,
|
||||
min_unindexed_rows: u64,
|
||||
grace: std::time::Duration,
|
||||
) -> Result<String> {
|
||||
let column = metadata_index::keys_of(dataset)
|
||||
.into_iter()
|
||||
.find(|k| k.key == key);
|
||||
match (action, column) {
|
||||
(Action::Add, None) => {
|
||||
if !metadata_index::requests(dataset).iter().any(|k| k == key) {
|
||||
return Ok(format!("{key} is no longer requested"));
|
||||
}
|
||||
let added = metadata_index::add_column(dataset, key).await?;
|
||||
if added.state != Some(State::Building) {
|
||||
return Ok(format!(
|
||||
"{key} was withdrawn while {} was added",
|
||||
added.column
|
||||
));
|
||||
}
|
||||
let filled = metadata_index::finish_after(dataset, &added, segment_rows, grace).await?;
|
||||
Ok(format!(
|
||||
"{key} indexed in {} ({filled} rows a racing write left derived again)",
|
||||
added.column
|
||||
))
|
||||
}
|
||||
(Action::Add, Some(column)) if column.state == Some(State::Building) => {
|
||||
// An add that stopped before setting again what its commit
|
||||
// overwrote leaves that to the build resuming it.
|
||||
let state = metadata_index::recover_add(dataset, &column).await?;
|
||||
if state != Some(State::Building) {
|
||||
return Ok(format!(
|
||||
"{key} is {} after its add was recovered",
|
||||
state.map_or("unknown", State::as_str)
|
||||
));
|
||||
}
|
||||
let filled =
|
||||
metadata_index::finish_after(dataset, &column, segment_rows, grace).await?;
|
||||
Ok(format!(
|
||||
"{key} indexed in {} ({filled} rows a racing write left derived again)",
|
||||
column.column
|
||||
))
|
||||
}
|
||||
(Action::Remove, Some(column)) if column.state == Some(State::Off) => {
|
||||
// Dropping the column drops its restore mark: set again first
|
||||
// what its add overwrote.
|
||||
metadata_index::recover_add(dataset, &column).await?;
|
||||
metadata_index::remove_column(dataset, &column.column).await?;
|
||||
Ok(format!("{key} removed with {}", column.column))
|
||||
}
|
||||
(Action::Maintain, Some(column)) if column.state == Some(State::Ready) => {
|
||||
let (filled, folded) =
|
||||
metadata_index::maintain(dataset, &column, segment_rows, min_unindexed_rows)
|
||||
.await?;
|
||||
Ok(format!("{key}: {folded:?}, {filled} rows derived again"))
|
||||
}
|
||||
(action, column) => Ok(format!(
|
||||
"{key} needs no {}: {}",
|
||||
action.as_str(),
|
||||
column
|
||||
.and_then(|c| c.state)
|
||||
.map_or("not indexed", State::as_str)
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds, maintains and removes the indexed metadata keys of S3 Vectors
|
||||
/// indexes, as each index's table records them. The admin dispatches a job
|
||||
/// when an operator asks; detection finds the rest.
|
||||
pub struct MetadataIndexHandler {
|
||||
namespace_url: String,
|
||||
storage: dataset::Storage,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
impl MetadataIndexHandler {
|
||||
pub fn new(namespace_url: String) -> Self {
|
||||
Self {
|
||||
namespace_url,
|
||||
storage: dataset::Storage::default(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
|
||||
self.metrics = metrics;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_storage(mut self, storage: dataset::Storage) -> Self {
|
||||
self.storage = storage;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn settings(values: &HashMap<String, ConfigValue>) -> (u64, u64) {
|
||||
(
|
||||
clamp(
|
||||
int_or(values, "segment_rows", DEFAULT_SEGMENT_ROWS as i64),
|
||||
SEGMENT_ROWS_FLOOR,
|
||||
SEGMENT_ROWS_CEILING,
|
||||
) as u64,
|
||||
clamp(
|
||||
int_or(
|
||||
values,
|
||||
"min_unindexed_rows",
|
||||
DEFAULT_MIN_UNINDEXED_ROWS as i64,
|
||||
),
|
||||
MIN_UNINDEXED_FLOOR,
|
||||
MIN_UNINDEXED_CEILING,
|
||||
) as u64,
|
||||
)
|
||||
}
|
||||
|
||||
fn string_param(parameters: &HashMap<String, ConfigValue>, name: &str) -> Option<String> {
|
||||
match parameters.get(name)?.kind.as_ref()? {
|
||||
Kind::StringValue(s) => Some(s.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for MetadataIndexHandler {
|
||||
fn capability(&self) -> JobTypeCapability {
|
||||
JobTypeCapability {
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
can_detect: true,
|
||||
can_execute: true,
|
||||
max_detection_concurrency: 1,
|
||||
max_execution_concurrency: 1,
|
||||
display_name: "Lance Metadata Index".to_string(),
|
||||
description: "Add, maintain and remove the indexed metadata keys of S3 Vectors indexes"
|
||||
.to_string(),
|
||||
weight: 40,
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor(&self) -> JobTypeDescriptor {
|
||||
let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
|
||||
defaults.insert(
|
||||
"segment_rows".to_string(),
|
||||
int_value(DEFAULT_SEGMENT_ROWS as i64),
|
||||
);
|
||||
defaults.insert(
|
||||
"min_unindexed_rows".to_string(),
|
||||
int_value(DEFAULT_MIN_UNINDEXED_ROWS as i64),
|
||||
);
|
||||
JobTypeDescriptor {
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
display_name: "Lance Metadata Index".to_string(),
|
||||
description:
|
||||
"Keep a LABEL_LIST index for each metadata key an S3 Vectors index asks for"
|
||||
.to_string(),
|
||||
icon: "fas fa-tags".to_string(),
|
||||
descriptor_version: 1,
|
||||
worker_config_form: Some(form(
|
||||
"lance-metadata-index-worker",
|
||||
"Metadata index",
|
||||
vec![
|
||||
number_field(
|
||||
"segment_rows",
|
||||
"Segment rows",
|
||||
"Rows an index segment takes before a new one starts",
|
||||
SEGMENT_ROWS_FLOOR,
|
||||
SEGMENT_ROWS_CEILING,
|
||||
),
|
||||
number_field(
|
||||
"min_unindexed_rows",
|
||||
"Unindexed rows",
|
||||
"Rows written since the last fold that wait for the next one",
|
||||
MIN_UNINDEXED_FLOOR,
|
||||
MIN_UNINDEXED_CEILING,
|
||||
),
|
||||
],
|
||||
defaults.clone(),
|
||||
)),
|
||||
worker_default_values: defaults,
|
||||
admin_runtime_defaults: Some(AdminRuntimeDefaults {
|
||||
job_type_max_runtime_seconds: EXECUTION_TIMEOUT_SECONDS,
|
||||
execution_timeout_seconds: EXECUTION_TIMEOUT_SECONDS,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn detect(
|
||||
&self,
|
||||
request: &RunDetectionRequest,
|
||||
sender: &dyn DetectionSender,
|
||||
) -> Result<()> {
|
||||
let (_, min_unindexed_rows) = settings(&request.worker_config_values);
|
||||
let max_results = usize::try_from(request.max_results).unwrap_or(0);
|
||||
let client = self.client();
|
||||
let mut proposals = Vec::new();
|
||||
let mut has_more = false;
|
||||
'tables: for encoded in &client.list_all_tables().await? {
|
||||
let id = parse_id(encoded);
|
||||
if !is_vector_table(&id) {
|
||||
continue;
|
||||
}
|
||||
let table = match dataset::open(&client, &id, &self.storage).await {
|
||||
Ok(table) => table,
|
||||
Err(err) => {
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.worker.object_skipped(JOB_TYPE, "open");
|
||||
}
|
||||
warn!("skipping {encoded}: {err:#}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.worker.object_seen(JOB_TYPE);
|
||||
}
|
||||
let work = match pending(&table.dataset, min_unindexed_rows).await {
|
||||
Ok(work) => work,
|
||||
Err(err) => {
|
||||
warn!("skipping {encoded}: reading its indexed keys failed: {err:#}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for (key, action) in work {
|
||||
if max_results > 0 && proposals.len() >= max_results {
|
||||
has_more = true;
|
||||
break 'tables;
|
||||
}
|
||||
proposals.push(proposal(&id, encoded, &key, action));
|
||||
}
|
||||
}
|
||||
let total = proposals.len() as i32;
|
||||
sender.send_proposals(DetectionProposals {
|
||||
request_id: request.request_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
proposals,
|
||||
has_more,
|
||||
})?;
|
||||
sender.send_complete(DetectionComplete {
|
||||
request_id: request.request_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
error_message: String::new(),
|
||||
total_proposals: total,
|
||||
skipped_reason: String::new(),
|
||||
eligible_after: None,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
request: &ExecuteJobRequest,
|
||||
sender: &dyn ExecutionSender,
|
||||
) -> Result<()> {
|
||||
let job = request
|
||||
.job
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow!("execute request carried no job"))?;
|
||||
let id = table_id(&job.parameters)
|
||||
.ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
|
||||
if !is_vector_table(&id) {
|
||||
bail!("{} is not an S3 Vectors index", id.join("/"));
|
||||
}
|
||||
let key = string_param(&job.parameters, "key")
|
||||
.ok_or_else(|| anyhow!("job {} carried no key", job.job_id))?;
|
||||
let action = string_param(&job.parameters, "action")
|
||||
.as_deref()
|
||||
.and_then(Action::parse)
|
||||
.ok_or_else(|| anyhow!("job {} carried no valid action", job.job_id))?;
|
||||
let (segment_rows, min_unindexed_rows) = settings(&request.worker_config_values);
|
||||
sender.send_progress(JobProgressUpdate {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
progress_percent: 10.0,
|
||||
stage: format!("{} {key}", action.as_str()),
|
||||
..Default::default()
|
||||
})?;
|
||||
let client = self.client();
|
||||
let mut table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let summary = run(
|
||||
&mut table.dataset,
|
||||
&key,
|
||||
action,
|
||||
segment_rows,
|
||||
min_unindexed_rows,
|
||||
metadata_index::RACE_GRACE,
|
||||
)
|
||||
.await?;
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
result: Some(JobResult {
|
||||
summary,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// The proposal for one key of one table; the admin builds the same
|
||||
/// parameters when it dispatches a job for an operator.
|
||||
fn proposal(id: &[String], encoded: &str, key: &str, action: Action) -> JobProposal {
|
||||
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
|
||||
parameters.insert("table_id".to_string(), string_list(id));
|
||||
parameters.insert("key".to_string(), string_value(key));
|
||||
parameters.insert("action".to_string(), string_value(action.as_str()));
|
||||
declare_build_runtime(&mut parameters);
|
||||
let dedupe = format!("{JOB_TYPE}:{encoded}:{key}:{}", action.as_str());
|
||||
JobProposal {
|
||||
proposal_id: dedupe.clone(),
|
||||
dedupe_key: dedupe,
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
summary: format!("{} metadata key {key} of {encoded}", action.as_str()),
|
||||
parameters,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_proposal_names_the_table_key_and_action() {
|
||||
let id = parse_id("media$vector$movies");
|
||||
let p = proposal(&id, "media$vector$movies", "author", Action::Remove);
|
||||
assert_eq!(table_id(&p.parameters), Some(id));
|
||||
assert_eq!(
|
||||
string_param(&p.parameters, "key").as_deref(),
|
||||
Some("author")
|
||||
);
|
||||
assert_eq!(
|
||||
string_param(&p.parameters, "action")
|
||||
.as_deref()
|
||||
.and_then(Action::parse),
|
||||
Some(Action::Remove)
|
||||
);
|
||||
assert_eq!(
|
||||
p.dedupe_key,
|
||||
"lance_metadata_index:media$vector$movies:author:remove"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_are_held_to_their_ranges() {
|
||||
let mut values = HashMap::new();
|
||||
assert_eq!(
|
||||
settings(&values),
|
||||
(DEFAULT_SEGMENT_ROWS, DEFAULT_MIN_UNINDEXED_ROWS)
|
||||
);
|
||||
values.insert("segment_rows".to_string(), int_value(-5));
|
||||
values.insert("min_unindexed_rows".to_string(), int_value(-1));
|
||||
assert_eq!(settings(&values), (SEGMENT_ROWS_FLOOR as u64, 0));
|
||||
}
|
||||
|
||||
/// A removal of a key whose add stopped before setting again what it
|
||||
/// overwrote sets that first: a key requested meanwhile stays requested.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_removal_keeps_what_the_add_overwrote() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut adder = dataset.clone();
|
||||
metadata_index::request(&mut dataset, "lang").await.unwrap();
|
||||
metadata_index::commit_column(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::request_removal(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
run(
|
||||
&mut adder,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::keys_of(&adder).is_empty());
|
||||
assert_eq!(metadata_index::requests(&adder), ["lang"]);
|
||||
}
|
||||
|
||||
/// A withdrawal of a request an interrupted add overwrote sets that
|
||||
/// first, so recovering the add does not bring the request back.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_withdrawal_of_a_request_an_add_overwrote_stays_withdrawn() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut adder = dataset.clone();
|
||||
metadata_index::request(&mut dataset, "lang").await.unwrap();
|
||||
metadata_index::commit_column(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
metadata_index::request_removal(&mut dataset, "lang")
|
||||
.await
|
||||
.unwrap();
|
||||
let author = metadata_index::keys_of(&dataset).remove(0);
|
||||
metadata_index::recover_add(&mut dataset, &author)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::requests(&dataset).is_empty());
|
||||
}
|
||||
|
||||
/// A withdrawal that read the request before the key's column was added
|
||||
/// and committed after sets the column off.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_withdrawal_racing_the_add_sets_the_column_off() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut operator = dataset.clone();
|
||||
let added = metadata_index::add_column(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(added.state, Some(State::Building));
|
||||
metadata_index::request_removal(&mut operator, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
|
||||
assert!(metadata_index::requests(&dataset).is_empty());
|
||||
}
|
||||
|
||||
/// A key that went ready on an empty table builds its first segment
|
||||
/// once rows arrive, below the threshold.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_key_ready_without_a_segment_is_maintained_at_once() {
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 1).await;
|
||||
dataset.delete("true").await.unwrap();
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
metadata_index::keys_of(&dataset)[0].state,
|
||||
Some(State::Ready)
|
||||
);
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
|
||||
let mut dataset = upsert(dataset, 0..100).await;
|
||||
assert_eq!(
|
||||
pending(&dataset, 1_000_000).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
1_000_000,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
metadata_index::segments(&dataset, "_mk_0")
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// What the table asks for is what a job does, and a job the table no
|
||||
/// longer asks for does nothing.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn jobs_follow_what_the_table_records() {
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::request(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pending(&dataset, 0).await.unwrap(),
|
||||
[
|
||||
("author".to_string(), Action::Add),
|
||||
("genre".to_string(), Action::Add)
|
||||
]
|
||||
);
|
||||
metadata_index::request_removal(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
let skipped = run(
|
||||
&mut dataset,
|
||||
"genre",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(skipped, "genre is no longer requested");
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = metadata_index::keys_of(&dataset);
|
||||
assert_eq!(keys.len(), 1);
|
||||
assert_eq!(keys[0].state, Some(State::Ready));
|
||||
assert_eq!(
|
||||
metadata_index::segments(&dataset, "_mk_0")
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
4
|
||||
);
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
|
||||
let mut dataset = upsert(dataset, 4_000..4_500).await;
|
||||
assert!(pending(&dataset, 1_000).await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
pending(&dataset, 500).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
500,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
|
||||
// A raced write that left NULL rows is maintained at once.
|
||||
let stale = dataset.clone();
|
||||
let mut dataset =
|
||||
crate::metadata_index::tests::lifecycle::upsert_with_null_columns(stale, 4_500..4_520)
|
||||
.await;
|
||||
assert_eq!(
|
||||
pending(&dataset, 1_000_000).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
1_000_000,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
let noop = run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(noop, "author needs no remove: ready");
|
||||
|
||||
metadata_index::request_removal(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pending(&dataset, 0).await.unwrap(),
|
||||
[("author".to_string(), Action::Remove)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::keys_of(&dataset).is_empty());
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,6 @@
|
||||
pub mod cleanup;
|
||||
pub mod compact;
|
||||
pub mod indices;
|
||||
pub mod metadata_index;
|
||||
pub mod purge;
|
||||
pub mod sort;
|
||||
pub mod vector_index;
|
||||
|
||||
@@ -67,12 +65,6 @@ pub fn handlers(
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
metadata_index::MetadataIndexHandler::new(namespace_url.clone())
|
||||
.with_storage(storage.clone())
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
sort::SortHandler::new(namespace_url.clone())
|
||||
.with_storage(storage.clone())
|
||||
|
||||
@@ -1,328 +0,0 @@
|
||||
//! Per-table retention for version cleanup, and purges: compaction that
|
||||
//! rewrites every fragment with deletions followed by a cleanup that keeps
|
||||
//! only the latest version, so the bytes of deleted rows leave storage now
|
||||
//! rather than after the retention window.
|
||||
//!
|
||||
//! Both are set in the table's own Lance config, so any client that can write
|
||||
//! the table can set them and they travel with it:
|
||||
//!
|
||||
//! - `seaweedfs.cleanup.retain_hours` and `seaweedfs.cleanup.min_versions`
|
||||
//! override the worker's retention for this table.
|
||||
//! - `seaweedfs.purge.requested` holds a request token ([`request_purge`]
|
||||
//! writes the current time); a purge is pending while it differs from
|
||||
//! `seaweedfs.purge.completed`, which the purge sets to the token it served.
|
||||
//! `seaweedfs.purge.rebuilt` records the token whose rebuild is done.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use chrono::Utc;
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{RemovalStats, cleanup_old_versions};
|
||||
use lance::dataset::optimize::{CompactionOptions, compact_files};
|
||||
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
|
||||
use lance_index::metrics::NoOpMetricsCollector;
|
||||
use lance_index::optimize::OptimizeOptions;
|
||||
use lance_index::{IndexType, is_system_index};
|
||||
use lance_table::format::IndexMetadata;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::jobs::cleanup::retention_policy;
|
||||
|
||||
pub const RETAIN_HOURS_KEY: &str = "seaweedfs.cleanup.retain_hours";
|
||||
pub const MIN_VERSIONS_KEY: &str = "seaweedfs.cleanup.min_versions";
|
||||
pub const PURGE_REQUESTED_KEY: &str = "seaweedfs.purge.requested";
|
||||
pub const PURGE_COMPLETED_KEY: &str = "seaweedfs.purge.completed";
|
||||
pub const PURGE_REBUILT_KEY: &str = "seaweedfs.purge.rebuilt";
|
||||
|
||||
/// The retention cleanup applies to one table: the worker's, unless the
|
||||
/// table's config overrides a value with one inside the worker's range.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Retention {
|
||||
pub retain_hours: i64,
|
||||
pub min_versions: usize,
|
||||
}
|
||||
|
||||
impl Retention {
|
||||
pub fn for_table(
|
||||
config: &HashMap<String, String>,
|
||||
worker: Retention,
|
||||
max_retain_hours: i64,
|
||||
max_min_versions: i64,
|
||||
) -> Retention {
|
||||
let read = |key: &str, min: i64, max: i64| -> Option<i64> {
|
||||
let raw = config.get(key)?;
|
||||
match raw.trim().parse::<i64>() {
|
||||
Ok(value) if (min..=max).contains(&value) => Some(value),
|
||||
_ => {
|
||||
warn!("ignoring table config {key}={raw:?}: not an integer in {min}..={max}");
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
Retention {
|
||||
retain_hours: read(RETAIN_HOURS_KEY, 0, max_retain_hours)
|
||||
.unwrap_or(worker.retain_hours),
|
||||
min_versions: read(MIN_VERSIONS_KEY, 1, max_min_versions)
|
||||
.map(|v| v as usize)
|
||||
.unwrap_or(worker.min_versions),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The request token of a purge the table is waiting for, if any.
|
||||
pub fn pending(config: &HashMap<String, String>) -> Option<String> {
|
||||
let requested = config.get(PURGE_REQUESTED_KEY).map(|s| s.trim())?;
|
||||
if requested.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match config.get(PURGE_COMPLETED_KEY) {
|
||||
Some(completed) if completed.trim() == requested => None,
|
||||
_ => Some(requested.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a purge request on the table and returns its token. The next
|
||||
/// cleanup detection proposes the purge.
|
||||
pub async fn request_purge(dataset: &mut Dataset) -> Result<String> {
|
||||
let token = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true);
|
||||
dataset
|
||||
.update_config([(PURGE_REQUESTED_KEY, token.as_str())])
|
||||
.await
|
||||
.context("record purge request")?;
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct PurgeStats {
|
||||
pub fragments_rewritten: usize,
|
||||
pub old_versions: u64,
|
||||
pub bytes_removed: u64,
|
||||
/// False while versions from before the rebuild remain.
|
||||
pub served: bool,
|
||||
}
|
||||
|
||||
/// Rewrites every fragment that has deleted rows and rebuilds the indices from
|
||||
/// the rows that remain, then removes the versions from before that through
|
||||
/// the cleanup job's retention policy with only the latest version kept, and
|
||||
/// marks `token` served once none of them is left. The rebuild is needed
|
||||
/// because compaction only remaps an index: an IVF_FLAT index keeps the
|
||||
/// deleted vector's values in its partitions under a remapped row address. It
|
||||
/// runs once per request, so a purge that has to wait for its cleanup (a
|
||||
/// version pinned by a tag, a query lease or one someone named, fails it)
|
||||
/// retries only the cleanup. Files no version references stay while Lance
|
||||
/// cannot tell them from a writer's uncommitted ones, which is up to seven
|
||||
/// days for an abandoned write.
|
||||
pub async fn purge(dataset: &mut Dataset, token: &str) -> Result<PurgeStats> {
|
||||
let fragments_rewritten = rebuild(dataset, token).await?;
|
||||
let cleanup = match retention_policy(dataset, 0, 1).await? {
|
||||
Some(policy) => cleanup_old_versions(dataset, policy)
|
||||
.await
|
||||
.context("remove versions older than the purge")?,
|
||||
None => RemovalStats::default(),
|
||||
};
|
||||
let served = finish(dataset, token).await?;
|
||||
Ok(PurgeStats {
|
||||
fragments_rewritten,
|
||||
old_versions: cleanup.old_versions,
|
||||
bytes_removed: cleanup.bytes_removed,
|
||||
served,
|
||||
})
|
||||
}
|
||||
|
||||
/// Rewrites the fragments with deletions and rebuilds the indices for
|
||||
/// `token`, unless that was done already, and returns how many fragments it
|
||||
/// rewrote.
|
||||
pub async fn rebuild(dataset: &mut Dataset, token: &str) -> Result<usize> {
|
||||
if dataset.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) == Some(token) {
|
||||
return Ok(0);
|
||||
}
|
||||
let vector_indices = vector_index_names(dataset).await?;
|
||||
let options = CompactionOptions {
|
||||
materialize_deletions: true,
|
||||
// Zero materializes every fragment with any deletion.
|
||||
materialize_deletions_threshold: 0.0,
|
||||
..Default::default()
|
||||
};
|
||||
let compaction = compact_files(dataset, options, None)
|
||||
.await
|
||||
.context("rewrite fragments with deletions")?;
|
||||
dataset.checkout_latest().await?;
|
||||
if !vector_indices.is_empty() {
|
||||
let mut options = OptimizeOptions::retrain();
|
||||
options.index_names = Some(vector_indices);
|
||||
dataset
|
||||
.optimize_indices(&options)
|
||||
.await
|
||||
.context("rebuild indices without the deleted rows")?;
|
||||
}
|
||||
rebuild_scalar_indices(dataset).await?;
|
||||
dataset.checkout_latest().await?;
|
||||
dataset
|
||||
.update_config([(PURGE_REBUILT_KEY, token)])
|
||||
.await
|
||||
.context("record purge rebuild")?;
|
||||
Ok(compaction.fragments_removed)
|
||||
}
|
||||
|
||||
/// Marks `token` served when the oldest version left is one the rebuild for
|
||||
/// it wrote, so no version from before the rebuild remains.
|
||||
pub async fn finish(dataset: &mut Dataset, token: &str) -> Result<bool> {
|
||||
let oldest = dataset
|
||||
.versions()
|
||||
.await?
|
||||
.iter()
|
||||
.map(|v| v.version)
|
||||
.min()
|
||||
.context("the table has no versions")?;
|
||||
let oldest = dataset.checkout_version(oldest).await?;
|
||||
if oldest.config().get(PURGE_REBUILT_KEY).map(|s| s.trim()) != Some(token) {
|
||||
return Ok(false);
|
||||
}
|
||||
dataset
|
||||
.update_config([(PURGE_COMPLETED_KEY, token)])
|
||||
.await
|
||||
.context("record purge completion")?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Names of the vector indices. A retrain builds each name over the whole
|
||||
/// table, so a column whose index is sharded over several names is refused:
|
||||
/// its shards would overlap.
|
||||
async fn vector_index_names(dataset: &Dataset) -> Result<Vec<String>> {
|
||||
let mut by_column: BTreeMap<Vec<i32>, BTreeSet<String>> = BTreeMap::new();
|
||||
for index in dataset
|
||||
.load_indices()
|
||||
.await?
|
||||
.iter()
|
||||
.filter(|i| is_vector(i))
|
||||
{
|
||||
by_column
|
||||
.entry(index.fields.clone())
|
||||
.or_default()
|
||||
.insert(index.name.clone());
|
||||
}
|
||||
let mut names = Vec::new();
|
||||
for shards in by_column.into_values() {
|
||||
if shards.len() > 1 {
|
||||
bail!(
|
||||
"the vector index is sharded over {shards:?}; a purge would rebuild each over the whole table"
|
||||
);
|
||||
}
|
||||
names.extend(shards);
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
/// Builds every scalar index again from the rows that remain. Compaction only
|
||||
/// remaps them: a bitmap index keeps a deleted key, and under stable row ids
|
||||
/// no scalar index is rewritten at all.
|
||||
async fn rebuild_scalar_indices(dataset: &mut Dataset) -> Result<()> {
|
||||
let indices = dataset.load_indices().await?;
|
||||
let mut rebuilt = BTreeSet::new();
|
||||
for index in indices.iter() {
|
||||
if is_system_index(index) || is_vector(index) || !rebuilt.insert(index.name.clone()) {
|
||||
continue;
|
||||
}
|
||||
let [field] = index.fields[..] else {
|
||||
bail!("index {} covers {} columns", index.name, index.fields.len());
|
||||
};
|
||||
let column = dataset.schema().field_path(field)?;
|
||||
let params = dataset
|
||||
.open_scalar_index(&column, &index.uuid, &NoOpMetricsCollector)
|
||||
.await?
|
||||
.derive_index_params()?;
|
||||
dataset
|
||||
.create_index(
|
||||
&[column.as_str()],
|
||||
IndexType::Scalar,
|
||||
Some(index.name.clone()),
|
||||
¶ms,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("rebuild index {}", index.name))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_vector(index: &IndexMetadata) -> bool {
|
||||
index
|
||||
.index_details
|
||||
.as_ref()
|
||||
.is_some_and(|details| details.type_url.ends_with("VectorIndexDetails"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config(pairs: &[(&str, &str)]) -> HashMap<String, String> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
const WORKER: Retention = Retention {
|
||||
retain_hours: 168,
|
||||
min_versions: 5,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn table_retention_overrides_within_range() {
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[]), WORKER, 8760, 1000),
|
||||
WORKER
|
||||
);
|
||||
assert_eq!(
|
||||
Retention::for_table(
|
||||
&config(&[(RETAIN_HOURS_KEY, "0"), (MIN_VERSIONS_KEY, "1")]),
|
||||
WORKER,
|
||||
8760,
|
||||
1000
|
||||
),
|
||||
Retention {
|
||||
retain_hours: 0,
|
||||
min_versions: 1
|
||||
}
|
||||
);
|
||||
for bad in ["-1", "8761", "x", ""] {
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[(RETAIN_HOURS_KEY, bad)]), WORKER, 8760, 1000)
|
||||
.retain_hours,
|
||||
168,
|
||||
"{bad:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
Retention::for_table(&config(&[(MIN_VERSIONS_KEY, "0")]), WORKER, 8760, 1000)
|
||||
.min_versions,
|
||||
5
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purge_is_pending_until_its_token_is_served() {
|
||||
assert_eq!(pending(&config(&[])), None);
|
||||
assert_eq!(pending(&config(&[(PURGE_REQUESTED_KEY, "")])), None);
|
||||
assert_eq!(
|
||||
pending(&config(&[(PURGE_REQUESTED_KEY, "t1")])),
|
||||
Some("t1".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
pending(&config(&[
|
||||
(PURGE_REQUESTED_KEY, "t2"),
|
||||
(PURGE_COMPLETED_KEY, "t1")
|
||||
])),
|
||||
Some("t2".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
pending(&config(&[
|
||||
(PURGE_REQUESTED_KEY, "t2"),
|
||||
(PURGE_COMPLETED_KEY, "t2")
|
||||
])),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5,18 +5,12 @@
|
||||
//! and its old versions removed. None can run in the Go worker, because all
|
||||
//! three read and rewrite Lance files. This crate is the worker that can.
|
||||
|
||||
// A job awaiting several Lance commits in turn needs deeper trait recursion
|
||||
// than the default to prove its future Send.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
pub mod catalog;
|
||||
pub mod cluster;
|
||||
pub mod dataset;
|
||||
pub mod direct_read;
|
||||
pub mod drop_guard;
|
||||
pub mod grpc_tls;
|
||||
pub mod jobs;
|
||||
pub mod metadata_index;
|
||||
pub mod metrics;
|
||||
pub mod preview;
|
||||
pub mod security;
|
||||
@@ -36,5 +30,3 @@ pub mod volume_server_pb {
|
||||
pub mod remote_pb {
|
||||
tonic::include_proto!("remote_pb");
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod drop_tests;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,394 +0,0 @@
|
||||
//! A purge removes a deleted vector's bytes from storage: after it, no file
|
||||
//! under the table holds the vector, its key or its metadata. Runs on a local
|
||||
//! table, so it needs no cluster.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::cleanup::{CleanupPolicy, RemovalStats, cleanup_old_versions};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance::index::vector::VectorIndexParams;
|
||||
use lance_index::IndexType;
|
||||
use lance_linalg::distance::DistanceType;
|
||||
use weed_lance_worker::jobs::purge;
|
||||
|
||||
const DIM: i32 = 8;
|
||||
const ROWS: usize = 300;
|
||||
const MARKER_ROW: usize = 137;
|
||||
const KEY_MARKER: &str = "purge-key-5f0c2e9a7d41b3c8";
|
||||
const METADATA_MARKER: &str = "purge-metadata-a93e1d5b77c2f046";
|
||||
|
||||
fn marker_vector() -> Vec<f32> {
|
||||
// Values no other row has, so their bytes identify the row.
|
||||
(0..DIM).map(|i| 31_337.125 + i as f32 * 0.5).collect()
|
||||
}
|
||||
|
||||
fn batch() -> (Arc<Schema>, RecordBatch) {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("key", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
|
||||
false,
|
||||
),
|
||||
Field::new("metadata", DataType::Utf8, true),
|
||||
]));
|
||||
let keys: Vec<String> = (0..ROWS)
|
||||
.map(|i| {
|
||||
if i == MARKER_ROW {
|
||||
KEY_MARKER.to_string()
|
||||
} else {
|
||||
format!("k{i}")
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let metadata: Vec<String> = (0..ROWS)
|
||||
.map(|i| {
|
||||
if i == MARKER_ROW {
|
||||
format!("{{\"note\":\"{METADATA_MARKER}\"}}")
|
||||
} else {
|
||||
format!("{{\"n\":{i}}}")
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
(0..ROWS).map(|i| {
|
||||
let values: Vec<f32> = if i == MARKER_ROW {
|
||||
marker_vector()
|
||||
} else {
|
||||
(0..DIM)
|
||||
.map(|d| ((i * 7 + d as usize) % 97) as f32)
|
||||
.collect()
|
||||
};
|
||||
Some(values.into_iter().map(Some).collect::<Vec<_>>())
|
||||
}),
|
||||
DIM,
|
||||
);
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(StringArray::from(keys)),
|
||||
Arc::new(vectors),
|
||||
Arc::new(StringArray::from(metadata)),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
(schema, batch)
|
||||
}
|
||||
|
||||
fn vector_bytes() -> Vec<u8> {
|
||||
marker_vector()
|
||||
.iter()
|
||||
.flat_map(|v| v.to_le_bytes())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Each marker found under `dir`, as "<marker> in <file>".
|
||||
fn markers_in(dir: &Path) -> Vec<String> {
|
||||
let vector = vector_bytes();
|
||||
let mut found = Vec::new();
|
||||
let mut stack = vec![dir.to_path_buf()];
|
||||
while let Some(path) = stack.pop() {
|
||||
for entry in std::fs::read_dir(&path).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_dir() {
|
||||
stack.push(entry.path());
|
||||
continue;
|
||||
}
|
||||
let bytes = std::fs::read(entry.path()).unwrap();
|
||||
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
|
||||
for (name, needle) in [
|
||||
("vector", vector.as_slice()),
|
||||
("key", KEY_MARKER.as_bytes()),
|
||||
("metadata", METADATA_MARKER.as_bytes()),
|
||||
] {
|
||||
if has(needle) {
|
||||
let file = entry.path();
|
||||
let file = file.strip_prefix(dir).unwrap_or(&file);
|
||||
found.push(format!("{name} in {}", file.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
found.sort_unstable();
|
||||
found
|
||||
}
|
||||
|
||||
/// A purge whose cleanup nothing holds back, so only the latest version
|
||||
/// stays. [`purge::purge`] cleans up through the cleanup job's retention
|
||||
/// policy, which a deployment may have keep more.
|
||||
async fn purge_to_latest(
|
||||
dataset: &mut Dataset,
|
||||
token: &str,
|
||||
) -> anyhow::Result<(usize, RemovalStats, bool)> {
|
||||
let rewritten = purge::rebuild(dataset, token).await?;
|
||||
let policy = CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now()),
|
||||
before_version: Some(dataset.latest_version_id().await?),
|
||||
error_if_tagged_old_versions: true,
|
||||
..Default::default()
|
||||
};
|
||||
let removed = cleanup_old_versions(dataset, policy).await?;
|
||||
Ok((rewritten, removed, purge::finish(dataset, token).await?))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_removes_a_deleted_vectors_bytes() {
|
||||
purge_leaves_no_trace(false).await;
|
||||
}
|
||||
|
||||
/// Under stable row ids compaction leaves scalar indices as they are, so only
|
||||
/// a rebuild takes the key out of them.
|
||||
#[tokio::test]
|
||||
async fn purge_removes_a_deleted_vectors_bytes_under_stable_row_ids() {
|
||||
purge_leaves_no_trace(true).await;
|
||||
}
|
||||
|
||||
async fn purge_leaves_no_trace(stable_row_ids: bool) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let params = lance::dataset::WriteParams {
|
||||
enable_stable_row_ids: stable_row_ids,
|
||||
..Default::default()
|
||||
};
|
||||
let mut dataset = Dataset::write(
|
||||
RecordBatchIterator::new(vec![Ok(batch)], schema),
|
||||
uri,
|
||||
Some(params),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
Some("vector_idx".to_string()),
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["key"],
|
||||
IndexType::BTree,
|
||||
Some("key_idx".to_string()),
|
||||
&lance_index::scalar::ScalarIndexParams::default(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// A remapped bitmap index still lists the deleted row's value.
|
||||
dataset
|
||||
.create_index(
|
||||
&["metadata"],
|
||||
IndexType::Bitmap,
|
||||
Some("metadata_idx".to_string()),
|
||||
&lance_index::scalar::ScalarIndexParams::for_builtin(
|
||||
lance_index::scalar::BuiltinIndexType::Bitmap,
|
||||
),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
|
||||
let before = markers_in(dir.path());
|
||||
assert!(
|
||||
before.iter().any(|m| m.starts_with("vector ")),
|
||||
"the vector's bytes should be found before deletion, found {before:?}"
|
||||
);
|
||||
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(dataset.count_rows(None).await.unwrap(), ROWS - 1);
|
||||
|
||||
// A deletion alone, and the default retention cleanup, leave the bytes:
|
||||
// the fragment still holds the row and older versions still reference it.
|
||||
cleanup_old_versions(
|
||||
&dataset,
|
||||
CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now() - chrono::Duration::hours(168)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
markers_in(dir.path())
|
||||
.iter()
|
||||
.any(|m| m.starts_with("vector ")),
|
||||
"a deleted vector is expected to stay until purged"
|
||||
);
|
||||
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert!(rewritten >= 1);
|
||||
assert!(removed.old_versions >= 1, "{removed:?}");
|
||||
|
||||
assert_eq!(
|
||||
markers_in(dir.path()),
|
||||
Vec::<String>::new(),
|
||||
"deleted vector's bytes remain in storage after the purge"
|
||||
);
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
|
||||
// The table is intact: every other row, and the index, still serve.
|
||||
let reopened = Dataset::open(uri).await.unwrap();
|
||||
assert_eq!(reopened.count_rows(None).await.unwrap(), ROWS - 1);
|
||||
assert_eq!(purge::pending(reopened.config()), None);
|
||||
assert_eq!(reopened.load_indices().await.unwrap().len(), 3);
|
||||
}
|
||||
|
||||
/// A version a query lease (a tag) pins still holds the deleted row, so the
|
||||
/// purge must fail and leave the request pending rather than report success.
|
||||
/// Once the tag goes, the next purge only cleans up: the rebuild is not done
|
||||
/// again.
|
||||
#[tokio::test]
|
||||
async fn purge_refuses_while_a_tag_pins_old_data() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.create_index(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
Some("vector_idx".to_string()),
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let pinned = dataset.version().version;
|
||||
dataset
|
||||
.tags()
|
||||
.create(
|
||||
"weed-vector-query-lease",
|
||||
lance::dataset::refs::Ref::Version(None, Some(pinned)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert!(purge_to_latest(&mut dataset, &token).await.is_err());
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
|
||||
let index = |dataset: &Dataset| {
|
||||
let dataset = dataset.clone();
|
||||
async move { dataset.load_indices().await.unwrap()[0].uuid }
|
||||
};
|
||||
let rebuilt = index(&dataset).await;
|
||||
dataset
|
||||
.tags()
|
||||
.delete("weed-vector-query-lease")
|
||||
.await
|
||||
.unwrap();
|
||||
let (rewritten, _, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert_eq!(rewritten, 0);
|
||||
assert_eq!(
|
||||
index(&dataset).await,
|
||||
rebuilt,
|
||||
"the index was rebuilt again"
|
||||
);
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
assert_eq!(markers_in(dir.path()), Vec::<String>::new());
|
||||
}
|
||||
|
||||
/// The request is served only once no version from before the rebuild is
|
||||
/// left, whatever kept one.
|
||||
#[tokio::test]
|
||||
async fn purge_is_served_once_older_versions_are_gone() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None)
|
||||
.await
|
||||
.unwrap();
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
assert_eq!(purge::rebuild(&mut dataset, &token).await.unwrap(), 1);
|
||||
assert!(!purge::finish(&mut dataset, &token).await.unwrap());
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token.clone()));
|
||||
|
||||
let (rewritten, removed, served) = purge_to_latest(&mut dataset, &token).await.unwrap();
|
||||
assert!(served);
|
||||
assert_eq!(rewritten, 0);
|
||||
assert!(removed.old_versions >= 1, "{removed:?}");
|
||||
assert_eq!(purge::pending(dataset.config()), None);
|
||||
}
|
||||
|
||||
/// A retrain builds every index name over the whole table, so shards of one
|
||||
/// vector index would end up overlapping: the purge refuses before touching
|
||||
/// the table and leaves the request pending.
|
||||
#[tokio::test]
|
||||
async fn purge_refuses_a_sharded_vector_index() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let uri = dir.path().join("index.lance");
|
||||
let uri = uri.to_str().unwrap();
|
||||
let (schema, batch) = batch();
|
||||
let params = lance::dataset::WriteParams {
|
||||
max_rows_per_file: ROWS / 2,
|
||||
..Default::default()
|
||||
};
|
||||
let mut dataset = Dataset::write(
|
||||
RecordBatchIterator::new(vec![Ok(batch)], schema),
|
||||
uri,
|
||||
Some(params),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let fragments: Vec<u32> = dataset
|
||||
.get_fragments()
|
||||
.iter()
|
||||
.map(|f| f.id() as u32)
|
||||
.collect();
|
||||
assert_eq!(fragments.len(), 2);
|
||||
for (shard, fragment) in fragments.iter().enumerate() {
|
||||
dataset
|
||||
.create_index_builder(
|
||||
&["vector"],
|
||||
IndexType::Vector,
|
||||
&VectorIndexParams::ivf_flat(2, DistanceType::L2),
|
||||
)
|
||||
.name(format!("shard-{shard}"))
|
||||
.fragments(vec![*fragment])
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
}
|
||||
dataset
|
||||
.delete(&format!("key = '{KEY_MARKER}'"))
|
||||
.await
|
||||
.unwrap();
|
||||
let token = purge::request_purge(&mut dataset).await.unwrap();
|
||||
let version = dataset.version().version;
|
||||
|
||||
let err = purge::purge(&mut dataset, &token).await.unwrap_err();
|
||||
assert!(format!("{err:#}").contains("sharded"), "{err:#}");
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(dataset.version().version, version, "the table was changed");
|
||||
assert_eq!(purge::pending(dataset.config()), Some(token));
|
||||
}
|
||||
@@ -13,9 +13,9 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/procfs v0.23.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.22.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/seaweedfs/seaweedfs => ../..
|
||||
+16
-12
@@ -2,31 +2,35 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
|
||||
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/procfs v0.23.0 h1:ad+S6FTkzwaMgacKLM5h6bNKExS+DMH0YiUwXItH/o4=
|
||||
github.com/prometheus/procfs v0.23.0/go.mod h1:3AVXO0FqLafyYFU/V/IQRZlC8Rgqz8Dm/x6pKLK+RIk=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
|
||||
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -74,15 +74,9 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
|
||||
|
||||
if limit > 0 && len(series.Versions) > limit {
|
||||
// Old releases are a long tail of one-cluster bands; keep the versions
|
||||
// most of the fleet is on now and sum the rest into one band. The
|
||||
// newest release is kept too, while it is still too small to rank, or
|
||||
// clusters upgrading to it look like they left the named versions.
|
||||
newest := newestRelease(series.Versions, last)
|
||||
// most of the fleet is on now and sum the rest into one band.
|
||||
ranked := append([]VersionCounts(nil), series.Versions...)
|
||||
sort.Slice(ranked, func(i, j int) bool {
|
||||
if (ranked[i].Version == newest) != (ranked[j].Version == newest) {
|
||||
return ranked[i].Version == newest
|
||||
}
|
||||
if ranked[i].Clusters[last] != ranked[j].Clusters[last] {
|
||||
return ranked[i].Clusters[last] > ranked[j].Clusters[last]
|
||||
}
|
||||
@@ -110,21 +104,6 @@ func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
|
||||
return series
|
||||
}
|
||||
|
||||
// newestRelease is the newest plain release, without a build suffix, that
|
||||
// clusters run on the last day.
|
||||
func newestRelease(versions []VersionCounts, last int) string {
|
||||
newest := ""
|
||||
for _, v := range versions {
|
||||
if _, suffix, ok := versionParts(v.Version); !ok || suffix != "" || v.Clusters[last] == 0 {
|
||||
continue
|
||||
}
|
||||
if newest == "" || versionLess(newest, v.Version) {
|
||||
newest = v.Version
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// versionedHistories drops the samples recorded before the reported version was
|
||||
// kept in history, so the version series spans the days it actually knows a
|
||||
// version for instead of climbing out of a run of blank days.
|
||||
|
||||
@@ -93,32 +93,6 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionSeriesKeepsNewestRelease(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
|
||||
seedSamples(s, "a", HistorySample{Version: "4.40"}, -2, -1, 0)
|
||||
seedSamples(s, "b", HistorySample{Version: "4.40"}, -2, -1, 0)
|
||||
seedSamples(s, "c", HistorySample{Version: "4.41"}, -2, -1, 0)
|
||||
seedSamples(s, "d", HistorySample{Version: "4.41"}, -2, -1, 0)
|
||||
// Upgraded to the newest release, which is still the smallest band.
|
||||
seedSamples(s, "e", HistorySample{Version: "4.41"}, -2)
|
||||
seedSamples(s, "e", HistorySample{Version: "4.42"}, -1, 0)
|
||||
// A suffixed build newer than every release does not take its place.
|
||||
seedSamples(s, "f", HistorySample{Version: "4.43-dev"}, -2, -1, 0)
|
||||
|
||||
series := s.GetVersionSeries(3, 2)
|
||||
var kept []string
|
||||
for _, v := range series.Versions {
|
||||
kept = append(kept, v.Version)
|
||||
}
|
||||
if len(kept) != 2 || kept[0] != "4.41" || kept[1] != "4.42" {
|
||||
t.Fatalf("kept = %v, want 4.41 and the newest release 4.42", kept)
|
||||
}
|
||||
if series.Other == nil || series.Other.Count != 2 {
|
||||
t.Fatalf("other = %+v, want 4.40 and 4.43-dev", series.Other)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionLess(t *testing.T) {
|
||||
// Ordered oldest to newest; every pair must compare in this order.
|
||||
ordered := []string{"unknown", "3.97", "4.02", "4.30", "4.30-enterprise", "9.99", "10.02"}
|
||||
|
||||
@@ -29,7 +29,6 @@ type FuseTestFramework struct {
|
||||
mountProcess *managedProcess
|
||||
filerAddr string
|
||||
filerPort int
|
||||
masterPort int
|
||||
weedBinary string
|
||||
isSetup bool
|
||||
config *TestConfig
|
||||
@@ -75,10 +74,6 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
require.NoError(t, err)
|
||||
|
||||
filerPort := freePort(t)
|
||||
masterPort := freePort(t)
|
||||
for masterPort == filerPort {
|
||||
masterPort = freePort(t)
|
||||
}
|
||||
|
||||
return &FuseTestFramework{
|
||||
t: t,
|
||||
@@ -88,7 +83,6 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
logDir: filepath.Join(tempDir, "logs"),
|
||||
filerPort: filerPort,
|
||||
filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort),
|
||||
masterPort: masterPort,
|
||||
weedBinary: findWeedBinary(),
|
||||
isSetup: false,
|
||||
config: config,
|
||||
@@ -224,11 +218,6 @@ func (f *FuseTestFramework) GetFilerAddr() string {
|
||||
return f.filerAddr
|
||||
}
|
||||
|
||||
// GetMasterAddr returns the master HTTP address of the mini cluster.
|
||||
func (f *FuseTestFramework) GetMasterAddr() string {
|
||||
return fmt.Sprintf("127.0.0.1:%d", f.masterPort)
|
||||
}
|
||||
|
||||
// managedProcess is a started weed sub-command whose exit is watched, so a wait
|
||||
// for it to come up ends the moment it dies instead of burning its full timeout.
|
||||
type managedProcess struct {
|
||||
@@ -332,7 +321,7 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
args = append(args, "-v=4")
|
||||
}
|
||||
adminPort := freePort(f.t)
|
||||
for adminPort == f.filerPort || adminPort == f.masterPort {
|
||||
for adminPort == f.filerPort {
|
||||
adminPort = freePort(f.t)
|
||||
}
|
||||
args = append(args,
|
||||
@@ -341,8 +330,6 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-filer.port="+strconv.Itoa(f.filerPort),
|
||||
"-master.port="+strconv.Itoa(f.masterPort),
|
||||
"-master.port.grpc="+strconv.Itoa(f.masterPort+grpcPortOffset),
|
||||
"-s3=false",
|
||||
"-webdav=false",
|
||||
"-admin.ui=false",
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
//go:build linux
|
||||
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
mergePageSize = 8192
|
||||
mergePages = 32
|
||||
)
|
||||
|
||||
// mergeHTTP bounds every HTTP call the test makes to the cluster.
|
||||
var mergeHTTP = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// mergeVacuumHTTP outlasts a master vacuum, which waits up to 30 s per volume
|
||||
// for writes assigned to it to drain before compacting it.
|
||||
var mergeVacuumHTTP = &http.Client{Timeout: 3 * time.Minute}
|
||||
|
||||
// TestReadOpenHandleAfterChunkMerge covers a file a flush merged while a
|
||||
// descriptor kept it open. The merge rewrites the file into new chunks and the
|
||||
// filer deletes the chunks it replaced, so after a vacuum only the new chunks
|
||||
// exist. Reads through the open handle must find them at once, whether or not
|
||||
// the handle has a write it has not flushed yet.
|
||||
func TestReadOpenHandleAfterChunkMerge(t *testing.T) {
|
||||
config := DefaultTestConfig()
|
||||
// The client chunk cache could still hold a replaced chunk; the reads
|
||||
// below have to reach the volume server.
|
||||
config.CacheSizeMB = 0
|
||||
config.MountOptions = append(config.MountOptions, "-cacheCapacityMB=0")
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("CleanHandle", func(t *testing.T) {
|
||||
testReadOpenHandleAfterChunkMerge(t, framework, "merged_clean.dat", false)
|
||||
})
|
||||
t.Run("HandleWithUnflushedWrite", func(t *testing.T) {
|
||||
testReadOpenHandleAfterChunkMerge(t, framework, "merged_dirty.dat", true)
|
||||
})
|
||||
}
|
||||
|
||||
func testReadOpenHandleAfterChunkMerge(t *testing.T, framework *FuseTestFramework, name string, unflushedWrite bool) {
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
held, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err)
|
||||
defer held.Close()
|
||||
|
||||
version := make([]int, mergePages)
|
||||
whole := make([]byte, 0, mergePages*mergePageSize)
|
||||
for p := 0; p < mergePages; p++ {
|
||||
whole = append(whole, mergePagePattern(p, 0)...)
|
||||
}
|
||||
_, err = held.WriteAt(whole, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, held.Sync())
|
||||
|
||||
// Overlapping 16-page windows: each leaves part of every older window
|
||||
// visible, so the stored chunks soon add up to more than twice the file
|
||||
// size and the next flush merges them.
|
||||
var replaced []mergeChunk
|
||||
for i, start := range []int{0, 3, 6, 9, 12, 15} {
|
||||
before := mergeEntryChunks(t, framework, name)
|
||||
window := make([]byte, 0, 16*mergePageSize)
|
||||
for p := start; p < start+16; p++ {
|
||||
window = append(window, mergePagePattern(p, i+1)...)
|
||||
version[p] = i + 1
|
||||
}
|
||||
_, err = held.WriteAt(window, int64(start*mergePageSize))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, held.Sync())
|
||||
if after := mergeEntryChunks(t, framework, name); len(after) == 1 && after[0].ModifiedTsNs == 0 {
|
||||
replaced = before
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, replaced, "the overlapping writes never made a flush merge the chunks")
|
||||
|
||||
// The filer deletes the replaced chunks after the merge commits; vacuum
|
||||
// once they are deleted so that only the merged chunk is left on disk.
|
||||
master := framework.GetMasterAddr()
|
||||
require.Eventually(t, func() bool {
|
||||
for _, c := range replaced {
|
||||
if mergeChunkStatus(master, c, false) != http.StatusNotFound {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 30*time.Second, 200*time.Millisecond, "the replaced chunks were never deleted")
|
||||
require.Eventually(t, func() bool {
|
||||
resp, err := mergeVacuumHTTP.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=0.0001", master))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
for _, c := range replaced {
|
||||
if mergeChunkStatus(master, c, true) != http.StatusNotFound {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 4*time.Minute, time.Second, "vacuum left the replaced chunks on disk")
|
||||
|
||||
if unflushedWrite {
|
||||
_, err = held.WriteAt(mergePagePattern(mergePages-1, 99), int64((mergePages-1)*mergePageSize))
|
||||
require.NoError(t, err)
|
||||
version[mergePages-1] = 99
|
||||
}
|
||||
|
||||
// A second descriptor shares the handle the first one keeps open. Drop its
|
||||
// pages from the kernel cache so every read goes to the mount.
|
||||
reader, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
require.NoError(t, unix.Fadvise(int(reader.Fd()), 0, 0, unix.FADV_DONTNEED))
|
||||
started := time.Now()
|
||||
buf := make([]byte, mergePageSize)
|
||||
for p := 0; p < mergePages; p++ {
|
||||
n, err := reader.ReadAt(buf, int64(p*mergePageSize))
|
||||
require.NoError(t, err, "read page %d through the open handle after the merge", p)
|
||||
require.Equal(t, mergePageSize, n, "page %d", p)
|
||||
require.Equal(t, mergePagePattern(p, version[p]), buf, "page %d", p)
|
||||
}
|
||||
assert.Less(t, time.Since(started), 5*time.Second, "reads through the open handle waited on chunks the merge replaced")
|
||||
}
|
||||
|
||||
func mergePagePattern(page, version int) []byte {
|
||||
b := make([]byte, mergePageSize)
|
||||
for i := 0; i < len(b); i += 8 {
|
||||
binary.LittleEndian.PutUint32(b[i:], uint32(page))
|
||||
binary.LittleEndian.PutUint32(b[i+4:], uint32(version))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type mergeChunk struct {
|
||||
FileId string `json:"file_id"`
|
||||
ModifiedTsNs int64 `json:"modified_ts_ns"`
|
||||
Fid *struct {
|
||||
VolumeId uint32 `json:"volume_id"`
|
||||
FileKey uint64 `json:"file_key"`
|
||||
Cookie uint32 `json:"cookie"`
|
||||
} `json:"fid"`
|
||||
}
|
||||
|
||||
func (c mergeChunk) fileId() string {
|
||||
if c.FileId != "" || c.Fid == nil {
|
||||
return c.FileId
|
||||
}
|
||||
return fmt.Sprintf("%d,%x%08x", c.Fid.VolumeId, c.Fid.FileKey, c.Fid.Cookie)
|
||||
}
|
||||
|
||||
// mergeEntryChunks returns the chunks the filer stores for the file.
|
||||
func mergeEntryChunks(t *testing.T, framework *FuseTestFramework, name string) []mergeChunk {
|
||||
t.Helper()
|
||||
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/%s?metadata=true", framework.GetFilerAddr(), url.PathEscape(name)))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "filer metadata of %s", name)
|
||||
var entry struct {
|
||||
Chunks []mergeChunk `json:"chunks"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&entry))
|
||||
return entry.Chunks
|
||||
}
|
||||
|
||||
// mergeChunkStatus HEADs a chunk on the volume server holding it, and returns
|
||||
// 0 when it cannot ask. With readDeleted, a deleted needle still answers until
|
||||
// a vacuum removes it.
|
||||
func mergeChunkStatus(master string, c mergeChunk, readDeleted bool) int {
|
||||
fid := c.fileId()
|
||||
volumeId := strings.SplitN(fid, ",", 2)[0]
|
||||
resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/dir/lookup?volumeId=%s", master, volumeId))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var lookup struct {
|
||||
Locations []struct {
|
||||
Url string `json:"url"`
|
||||
} `json:"locations"`
|
||||
}
|
||||
err = json.NewDecoder(resp.Body).Decode(&lookup)
|
||||
resp.Body.Close()
|
||||
if err != nil || len(lookup.Locations) == 0 {
|
||||
return 0
|
||||
}
|
||||
target := fmt.Sprintf("http://%s/%s", lookup.Locations[0].Url, fid)
|
||||
if readDeleted {
|
||||
target += "?readDeleted=true"
|
||||
}
|
||||
head, err := mergeHTTP.Head(target)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
head.Body.Close()
|
||||
return head.StatusCode
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package metadata_subscribe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// A peer removal is broadcast only to the filers connected to the master that
|
||||
// saw the peer go. A peer that dies while no master is up (or across a leader
|
||||
// change) is never removed: the filer keeps retrying its address, the peer's
|
||||
// delivery watermark stays at zero, and every aggregated subscriber of that
|
||||
// filer is held at the settled horizon (2 x LogFlushInterval) for good.
|
||||
func TestAggregatedSubscriberNotHeldByPeerThatDiedUnnoticed(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
weedBinary := findWeedBinary()
|
||||
require.NotEmpty(t, weedBinary, "weed binary not found")
|
||||
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_peer_unreachable_")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
if t.Failed() {
|
||||
t.Logf("logs kept at %s", testDir)
|
||||
return
|
||||
}
|
||||
os.RemoveAll(testDir)
|
||||
})
|
||||
|
||||
ports, err := testutil.AllocateMiniPorts(3)
|
||||
require.NoError(t, err)
|
||||
masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
|
||||
master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
|
||||
filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
|
||||
|
||||
masterArgs := []string{"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(masterPort),
|
||||
"-mdir=" + mkdir(t, testDir, "master"),
|
||||
"-peers=none"}
|
||||
masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
||||
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
||||
startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
|
||||
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
|
||||
filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
|
||||
require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
|
||||
filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
|
||||
|
||||
// filer1 follows filer2: an entry created on filer2 reaches filer1
|
||||
createPeerEntry(t, filer2Address, "from-filer2")
|
||||
require.NoError(t, waitForPeerEntry(filer1Address, "from-filer2", 60*time.Second),
|
||||
"filer1 never followed filer2")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
events := make(chan *filer_pb.SubscribeMetadataResponse, 1000)
|
||||
sinceNs := time.Now().UnixNano()
|
||||
go subscribeToMetadataWithOptions(ctx, filer1Address, peerEntryDir, sinceNs, events)
|
||||
arrived := make(map[string]time.Time)
|
||||
waitArrival := func(name string, within time.Duration) bool {
|
||||
deadline := time.After(within)
|
||||
for {
|
||||
if _, ok := arrived[name]; ok {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case ev := <-events:
|
||||
if n := ev.GetEventNotification().GetNewEntry(); n != nil {
|
||||
arrived[n.Name] = time.Now()
|
||||
}
|
||||
case <-deadline:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
createPeerEntry(t, filer1Address, "baseline")
|
||||
require.True(t, waitArrival("baseline", 20*time.Second), "the baseline event never reached the subscriber")
|
||||
|
||||
// filer2 dies while no master is up, so no removal is ever broadcast
|
||||
stopProcess(masterProcess)
|
||||
stopProcess(filer2)
|
||||
startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
|
||||
require.NoError(t, waitForLeader(masterPort, 60*time.Second))
|
||||
require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second))
|
||||
diedAt := time.Now()
|
||||
|
||||
// Unpatched, every event from here on arrives 2 x LogFlushInterval late.
|
||||
// The dead peer may hold reads for its unreachable bound plus the removal
|
||||
// grace (2 x LogFlushInterval each); after that delivery must be prompt.
|
||||
for i := 0; ; i++ {
|
||||
name := fmt.Sprintf("probe-%d", i)
|
||||
createPeerEntry(t, filer1Address, name)
|
||||
if waitArrival(name, 10*time.Second) {
|
||||
t.Logf("%s delivered promptly %v after the peer died", name, time.Since(diedAt).Round(time.Second))
|
||||
return
|
||||
}
|
||||
if time.Since(diedAt) > 6*time.Minute {
|
||||
t.Fatalf("aggregated delivery still held %v after the peer died unnoticed", time.Since(diedAt).Round(time.Second))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,16 +2,13 @@ package acl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -66,22 +63,12 @@ func TestBucketOwnershipControlsLifecycle(t *testing.T) {
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
|
||||
deleted, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
raw, ok := awsmiddleware.GetRawResponse(deleted.ResultMetadata).(*smithyhttp.Response)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, http.StatusNoContent, raw.StatusCode)
|
||||
|
||||
_, err = client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "OwnershipControlsNotFoundError")
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerPreferred)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerPreferred, getObjectOwnership(t, client, bucketName))
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
}
|
||||
|
||||
// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
|
||||
|
||||
Loaded 100 of 476 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user